{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Description:\n",
    "这里用Pytorch搭建DeepFM Model在cretio数据集上进行点击率预测的任务"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T11:49:14.994656Z",
     "start_time": "2020-12-28T11:49:14.706429Z"
    }
   },
   "outputs": [],
   "source": [
    "\"\"\"导入包\"\"\"\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm\n",
    "import datetime\n",
    "\n",
    "# pytorch\n",
    "import torch\n",
    "from torch.utils.data import DataLoader, Dataset, TensorDataset\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "from torchkeras import summary, Model\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T11:49:41.513722Z",
     "start_time": "2020-12-28T11:49:41.500709Z"
    }
   },
   "outputs": [],
   "source": [
    "file_path = './preprocessed_data/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T11:58:10.669818Z",
     "start_time": "2020-12-28T11:58:10.664832Z"
    }
   },
   "outputs": [],
   "source": [
    "def prepared_data(file_path):\n",
    "    # 读入训练集， 验证集和测试集\n",
    "    train = pd.read_csv(file_path + 'train_set.csv')\n",
    "    val = pd.read_csv(file_path + 'val_set.csv')\n",
    "    test = pd.read_csv(file_path + 'test_set.csv')\n",
    "    \n",
    "    trn_x, trn_y = train.drop(columns='Label').values, train['Label'].values\n",
    "    val_x, val_y = val.drop(columns='Label').values, val['Label'].values\n",
    "    test_x = test.values\n",
    "    \n",
    "    fea_col = np.load(file_path + 'fea_col.npy', allow_pickle=True)\n",
    "    \n",
    "    return fea_col, (trn_x, trn_y), (val_x, val_y), test_x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T11:58:12.998370Z",
     "start_time": "2020-12-28T11:58:12.966456Z"
    }
   },
   "outputs": [],
   "source": [
    "\"\"\"导入数据\"\"\"\n",
    "fea_cols, (trn_x, trn_y), (val_x, val_y), test_x = prepared_data(file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T11:58:16.017381Z",
     "start_time": "2020-12-28T11:58:15.996436Z"
    }
   },
   "outputs": [],
   "source": [
    "# 把数据构建成数据管道\n",
    "dl_train_dataset = TensorDataset(torch.tensor(trn_x).float(), torch.tensor(trn_y).float())\n",
    "dl_val_dataset = TensorDataset(torch.tensor(val_x).float(), torch.tensor(val_y).float())\n",
    "\n",
    "dl_train = DataLoader(dl_train_dataset, shuffle=True, batch_size=32)\n",
    "dl_val = DataLoader(dl_val_dataset, shuffle=True, batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T11:58:58.043806Z",
     "start_time": "2020-12-28T11:58:58.034774Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([32, 39]) tensor([0., 0., 0., 1., 0., 0., 1., 1., 0., 0., 1., 0., 1., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 1., 0., 0., 0.])\n"
     ]
    }
   ],
   "source": [
    "# 看一眼数据\n",
    "for x, y in iter(dl_train):\n",
    "    print(x.shape, y)\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 构建模型\n",
    "这里依然是使用继承nn.Modult基类构建模型， 并辅助应用模型容器进行封装， DeepFM的模型架构如下：\n",
    "\n",
    "![](https://img-blog.csdnimg.cn/20201225092253200.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3d1emhvbmdxaWFuZw==,size_1,color_FFFFFF,t_70#pic_center)\n",
    "\n",
    "这个模型也是两部分组成， 左边是FM模型， 右边是DNN模型， DNN模型和之前的形式一样， 所以下面我们首先先实现这两个单模型，然后把它们拼起来"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T13:29:47.674211Z",
     "start_time": "2020-12-28T13:29:47.654173Z"
    }
   },
   "outputs": [],
   "source": [
    "class FM(nn.Module):\n",
    "    \"\"\"FM part\"\"\"\n",
    "    def __init__(self, latent_dim, fea_num):\n",
    "        \"\"\"\n",
    "        latent_dim: 各个离散特征隐向量的维度\n",
    "        input_shape: 这个最后离散特征embedding之后的拼接和dense拼接的总特征个数\n",
    "        \"\"\"\n",
    "        super(FM, self).__init__()\n",
    "        \n",
    "        self.latent_dim = latent_dim\n",
    "        # 定义三个矩阵， 一个是全局偏置，一个是一阶权重矩阵， 一个是二阶交叉矩阵，注意这里的参数由于是可学习参数，需要用nn.Parameter进行定义\n",
    "        self.w0 = nn.Parameter(torch.zeros([1,]))\n",
    "        self.w1 = nn.Parameter(torch.rand([fea_num, 1]))\n",
    "        self.w2 = nn.Parameter(torch.rand([fea_num, latent_dim]))\n",
    "        \n",
    "    def forward(self, inputs):   \n",
    "        # 一阶交叉\n",
    "        first_order = self.w0 + torch.mm(inputs, self.w1)      # (samples_num, 1)\n",
    "        # 二阶交叉  这个用FM的最终化简公式\n",
    "        second_order = 1/2 * torch.sum(\n",
    "            torch.pow(torch.mm(inputs, self.w2), 2) - torch.mm(torch.pow(inputs,2), torch.pow(self.w2, 2)),\n",
    "            dim = 1,\n",
    "            keepdim = True\n",
    "        )         # (samples_num, 1)\n",
    "        \n",
    "        return first_order + second_order "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T13:29:17.311726Z",
     "start_time": "2020-12-28T13:29:17.298718Z"
    }
   },
   "outputs": [],
   "source": [
    "class Dnn(nn.Module):\n",
    "    \"\"\"Dnn part\"\"\"\n",
    "    def __init__(self, hidden_units, dropout=0.):\n",
    "        \"\"\"\n",
    "        hidden_units: 列表， 每个元素表示每一层的神经单元个数， 比如[256, 128, 64], 两层网络， 第一层神经单元128， 第二层64， 第一个维度是输入维度\n",
    "        dropout = 0.\n",
    "        \"\"\"\n",
    "        super(Dnn, self).__init__()\n",
    "        \n",
    "        self.dnn_network = nn.ModuleList([nn.Linear(layer[0], layer[1]) for layer in list(zip(hidden_units[:-1], hidden_units[1:]))])\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "    \n",
    "    def forward(self, x):  \n",
    "        for linear in self.dnn_network:\n",
    "            x = linear(x)\n",
    "            x = F.relu(x)    \n",
    "        x = self.dropout(x) \n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T13:30:18.314791Z",
     "start_time": "2020-12-28T13:30:18.305816Z"
    }
   },
   "outputs": [],
   "source": [
    "class DeepFM(nn.Module):\n",
    "    def __init__(self, feature_columns, hidden_units, dnn_dropout=0.):\n",
    "        \"\"\"\n",
    "        DeepFM:\n",
    "        :param feature_columns: 特征信息， 这个传入的是fea_cols\n",
    "        :param hidden_units: 隐藏单元个数， 一个列表的形式， 列表的长度代表层数， 每个元素代表每一层神经元个数\n",
    "        \"\"\"\n",
    "        super(DeepFM, self).__init__()\n",
    "        self.dense_feature_cols, self.sparse_feature_cols = feature_columns\n",
    "        \n",
    "        # embedding\n",
    "        self.embed_layers = nn.ModuleDict({\n",
    "            'embed_' + str(i): nn.Embedding(num_embeddings=feat['feat_num'], embedding_dim=feat['embed_dim'])\n",
    "            for i, feat in enumerate(self.sparse_feature_cols)\n",
    "        })\n",
    "        \n",
    "        # 这里要注意Pytorch的linear和tf的dense的不同之处， 前者的linear需要输入特征和输出特征维度， 而传入的hidden_units的第一个是第一层隐藏的神经单元个数，这里需要加个输入维度\n",
    "        self.fea_num = len(self.dense_feature_cols) + len(self.sparse_feature_cols)*self.sparse_feature_cols[0]['embed_dim']\n",
    "        hidden_units.insert(0, self.fea_num)\n",
    "        \n",
    "        self.fm = FM(self.sparse_feature_cols[0]['embed_dim'], self.fea_num)     \n",
    "        self.dnn_network = Dnn(hidden_units, dnn_dropout)\n",
    "        self.nn_final_linear = nn.Linear(hidden_units[-1], 1)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        dense_inputs, sparse_inputs = x[:, :len(self.dense_feature_cols)], x[:, len(self.dense_feature_cols):]\n",
    "        sparse_inputs = sparse_inputs.long()       # 转成long类型才能作为nn.embedding的输入\n",
    "        sparse_embeds = [self.embed_layers['embed_'+str(i)](sparse_inputs[:, i]) for i in range(sparse_inputs.shape[1])]\n",
    "        sparse_embeds = torch.cat(sparse_embeds, dim=-1)\n",
    "        \n",
    "        # 把离散特征和连续特征进行拼接作为FM和DNN的输入\n",
    "        x = torch.cat([sparse_embeds, dense_inputs], dim=-1)\n",
    "        # Wide\n",
    "        wide_outputs = self.fm(x)\n",
    "        # deep\n",
    "        deep_outputs = self.nn_final_linear(self.dnn_network(x))\n",
    "        \n",
    "        # 模型的最后输出\n",
    "        outputs = F.sigmoid(torch.add(wide_outputs, deep_outputs))\n",
    "        \n",
    "        return outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T13:30:20.817806Z",
     "start_time": "2020-12-28T13:30:20.809784Z"
    }
   },
   "outputs": [],
   "source": [
    "# 建立模型\n",
    "hidden_units = [128, 64, 32]\n",
    "dnn_dropout = 0.\n",
    "\n",
    "model = DeepFM(fea_cols, hidden_units, dnn_dropout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T13:30:21.615630Z",
     "start_time": "2020-12-28T13:30:21.598675Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "         Embedding-1                    [-1, 8]             632\n",
      "         Embedding-2                    [-1, 8]           2,016\n",
      "         Embedding-3                    [-1, 8]          10,344\n",
      "         Embedding-4                    [-1, 8]           8,344\n",
      "         Embedding-5                    [-1, 8]             240\n",
      "         Embedding-6                    [-1, 8]              56\n",
      "         Embedding-7                    [-1, 8]           9,312\n",
      "         Embedding-8                    [-1, 8]             312\n",
      "         Embedding-9                    [-1, 8]              16\n",
      "        Embedding-10                    [-1, 8]           7,264\n",
      "        Embedding-11                    [-1, 8]           7,408\n",
      "        Embedding-12                    [-1, 8]           9,912\n",
      "        Embedding-13                    [-1, 8]           6,592\n",
      "        Embedding-14                    [-1, 8]             160\n",
      "        Embedding-15                    [-1, 8]           6,552\n",
      "        Embedding-16                    [-1, 8]           9,272\n",
      "        Embedding-17                    [-1, 8]              72\n",
      "        Embedding-18                    [-1, 8]           4,272\n",
      "        Embedding-19                    [-1, 8]           1,608\n",
      "        Embedding-20                    [-1, 8]              32\n",
      "        Embedding-21                    [-1, 8]           9,632\n",
      "        Embedding-22                    [-1, 8]              56\n",
      "        Embedding-23                    [-1, 8]              96\n",
      "        Embedding-24                    [-1, 8]           5,832\n",
      "        Embedding-25                    [-1, 8]             264\n",
      "        Embedding-26                    [-1, 8]           4,432\n",
      "               FM-27                    [-1, 1]           1,990\n",
      "           Linear-28                  [-1, 128]          28,416\n",
      "           Linear-29                   [-1, 64]           8,256\n",
      "           Linear-30                   [-1, 32]           2,080\n",
      "          Dropout-31                   [-1, 32]               0\n",
      "           Linear-32                    [-1, 1]              33\n",
      "================================================================\n",
      "Total params: 145,503\n",
      "Trainable params: 145,503\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.000149\n",
      "Forward/backward pass size (MB): 0.003555\n",
      "Params size (MB): 0.555050\n",
      "Estimated Total Size (MB): 0.558754\n",
      "----------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "summary(model, input_shape=(trn_x.shape[1],))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T13:31:03.464822Z",
     "start_time": "2020-12-28T13:31:03.440886Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.0000e+00],\n",
      "        [0.0000e+00],\n",
      "        [1.0000e+00],\n",
      "        [9.1855e-01],\n",
      "        [0.0000e+00],\n",
      "        [0.0000e+00],\n",
      "        [0.0000e+00],\n",
      "        [0.0000e+00],\n",
      "        [0.0000e+00],\n",
      "        [0.0000e+00],\n",
      "        [1.0000e+00],\n",
      "        [7.5162e-16],\n",
      "        [0.0000e+00],\n",
      "        [0.0000e+00],\n",
      "        [0.0000e+00],\n",
      "        [0.0000e+00],\n",
      "        [1.0000e+00],\n",
      "        [2.4068e-22],\n",
      "        [0.0000e+00],\n",
      "        [2.8100e-02],\n",
      "        [4.9510e-36],\n",
      "        [0.0000e+00],\n",
      "        [1.0000e+00],\n",
      "        [0.0000e+00],\n",
      "        [1.0000e+00],\n",
      "        [1.5273e-22],\n",
      "        [0.0000e+00],\n",
      "        [0.0000e+00],\n",
      "        [0.0000e+00],\n",
      "        [0.0000e+00],\n",
      "        [1.0000e+00],\n",
      "        [1.6803e-25]], grad_fn=<SigmoidBackward>)\n"
     ]
    }
   ],
   "source": [
    "# 测试一下模型\n",
    "for fea, label in iter(dl_train):\n",
    "    out = model(fea)\n",
    "    print(out)\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型的训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T13:32:59.416192Z",
     "start_time": "2020-12-28T13:32:59.406218Z"
    }
   },
   "outputs": [],
   "source": [
    "# 模型的相关配置\n",
    "def auc(y_pred, y_true):\n",
    "    pred = y_pred.data\n",
    "    y = y_true.data\n",
    "    return roc_auc_score(y, pred)\n",
    "\n",
    "loss_func = nn.BCELoss()\n",
    "optimizer = optim.Adam(params=model.parameters(), lr=0.001)\n",
    "metric_func = auc\n",
    "metric_name = 'auc'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T13:33:43.863241Z",
     "start_time": "2020-12-28T13:33:35.202257Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start_training.........\n",
      "================================================================2020-12-28 21:33:35\n",
      "[step=10] loss: 9.399, auc: 0.537\n",
      "[step=20] loss: 9.084, auc: 0.539\n",
      "[step=30] loss: 9.249, auc: 0.527\n",
      "[step=40] loss: 9.540, auc: 0.523\n",
      "\n",
      "EPOCH=1, loss=9.540, auc = 0.523, val_loss=9.009, val_auc = 0.508\n",
      "\n",
      "================================================================================2020-12-28 21:33:37\n",
      "[step=10] loss: 8.839, auc: 0.530\n",
      "[step=20] loss: 9.327, auc: 0.531\n",
      "[step=30] loss: 9.717, auc: 0.524\n",
      "[step=40] loss: 9.282, auc: 0.526\n",
      "\n",
      "EPOCH=2, loss=9.282, auc = 0.526, val_loss=9.060, val_auc = 0.511\n",
      "\n",
      "================================================================================2020-12-28 21:33:37\n",
      "[step=10] loss: 8.844, auc: 0.523\n",
      "[step=20] loss: 9.105, auc: 0.536\n",
      "[step=30] loss: 9.053, auc: 0.538\n",
      "[step=40] loss: 9.134, auc: 0.537\n",
      "\n",
      "EPOCH=3, loss=9.134, auc = 0.537, val_loss=8.907, val_auc = 0.502\n",
      "\n",
      "================================================================================2020-12-28 21:33:38\n",
      "[step=10] loss: 9.444, auc: 0.523\n",
      "[step=20] loss: 9.094, auc: 0.525\n",
      "[step=30] loss: 8.490, auc: 0.536\n",
      "[step=40] loss: 8.736, auc: 0.539\n",
      "\n",
      "EPOCH=4, loss=8.736, auc = 0.539, val_loss=8.213, val_auc = 0.520\n",
      "\n",
      "================================================================================2020-12-28 21:33:39\n",
      "[step=10] loss: 8.530, auc: 0.586\n",
      "[step=20] loss: 8.402, auc: 0.570\n",
      "[step=30] loss: 8.225, auc: 0.553\n",
      "[step=40] loss: 8.369, auc: 0.545\n",
      "\n",
      "EPOCH=5, loss=8.369, auc = 0.545, val_loss=8.475, val_auc = 0.513\n",
      "\n",
      "================================================================================2020-12-28 21:33:39\n",
      "[step=10] loss: 7.934, auc: 0.585\n",
      "[step=20] loss: 8.496, auc: 0.548\n",
      "[step=30] loss: 8.213, auc: 0.549\n",
      "[step=40] loss: 8.119, auc: 0.552\n",
      "\n",
      "EPOCH=6, loss=8.119, auc = 0.552, val_loss=8.693, val_auc = 0.512\n",
      "\n",
      "================================================================================2020-12-28 21:33:40\n",
      "[step=10] loss: 8.084, auc: 0.529\n",
      "[step=20] loss: 7.645, auc: 0.534\n",
      "[step=30] loss: 7.588, auc: 0.562\n",
      "[step=40] loss: 7.687, auc: 0.559\n",
      "\n",
      "EPOCH=7, loss=7.687, auc = 0.559, val_loss=7.955, val_auc = 0.541\n",
      "\n",
      "================================================================================2020-12-28 21:33:41\n",
      "[step=10] loss: 7.026, auc: 0.525\n",
      "[step=20] loss: 7.001, auc: 0.578\n",
      "[step=30] loss: 7.349, auc: 0.566\n",
      "[step=40] loss: 7.577, auc: 0.566\n",
      "\n",
      "EPOCH=8, loss=7.577, auc = 0.566, val_loss=8.241, val_auc = 0.575\n",
      "\n",
      "================================================================================2020-12-28 21:33:42\n",
      "[step=10] loss: 6.536, auc: 0.591\n",
      "[step=20] loss: 6.854, auc: 0.582\n",
      "[step=30] loss: 7.128, auc: 0.572\n",
      "[step=40] loss: 7.217, auc: 0.568\n",
      "\n",
      "EPOCH=9, loss=7.217, auc = 0.568, val_loss=6.733, val_auc = 0.522\n",
      "\n",
      "================================================================================2020-12-28 21:33:43\n",
      "[step=10] loss: 8.611, auc: 0.556\n",
      "[step=20] loss: 7.866, auc: 0.562\n",
      "[step=30] loss: 7.241, auc: 0.579\n",
      "[step=40] loss: 7.297, auc: 0.577\n",
      "\n",
      "EPOCH=10, loss=7.297, auc = 0.577, val_loss=5.848, val_auc = 0.507\n",
      "\n",
      "================================================================================2020-12-28 21:33:43\n",
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "# 脚本训练风格\n",
    "epochs = 10\n",
    "log_step_freq = 10\n",
    "\n",
    "dfhistory = pd.DataFrame(columns=['epoch', 'loss', metric_name, 'val_loss', 'val_'+metric_name])\n",
    "\n",
    "print('start_training.........')\n",
    "nowtime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')\n",
    "print('========'*8 + '%s' %nowtime)\n",
    "\n",
    "for epoch in range(1, epochs+1):\n",
    "    \n",
    "    # 训练阶段\n",
    "    model.train()\n",
    "    loss_sum = 0.0\n",
    "    metric_sum = 0.0\n",
    "    step = 1\n",
    "    \n",
    "    for step, (features, labels) in enumerate(dl_train, 1):\n",
    "        # 梯度清零\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # 正向传播\n",
    "        predictions = model(features);\n",
    "        loss = loss_func(predictions, labels)\n",
    "        try:\n",
    "            metric = metric_func(predictions, labels)\n",
    "        except ValueError:\n",
    "            pass\n",
    "        \n",
    "        # 反向传播\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        # 打印batch级别日志\n",
    "        loss_sum += loss.item()\n",
    "        metric_sum += metric.item()\n",
    "        if step % log_step_freq == 0:\n",
    "            print((\"[step=%d] loss: %.3f, \" + metric_name + \": %.3f\") % (step, loss_sum/step, metric_sum/step));\n",
    "    \n",
    "    # 验证阶段\n",
    "    model.eval()\n",
    "    val_loss_sum = 0.0\n",
    "    val_metric_sum = 0.0\n",
    "    val_step = 1\n",
    "    \n",
    "    for val_step, (features, labels) in enumerate(dl_val, 1):\n",
    "        with torch.no_grad():\n",
    "            predictions = model(features)\n",
    "            val_loss = loss_func(predictions, labels)\n",
    "            try:\n",
    "                val_metric = metric_func(predictions, labels)\n",
    "            except ValueError:\n",
    "                pass\n",
    "        \n",
    "        val_loss_sum += val_loss.item()\n",
    "        val_metric_sum += val_metric.item()\n",
    "    \n",
    "    # 记录日志\n",
    "    info = (epoch, loss_sum/step, metric_sum/step, val_loss_sum/val_step, val_metric_sum/val_step)\n",
    "    dfhistory.loc[epoch-1] = info\n",
    "    \n",
    "    # 打印日志\n",
    "    print((\"\\nEPOCH=%d, loss=%.3f, \" + metric_name + \" = %.3f, val_loss=%.3f, \" + \"val_\" + metric_name + \" = %.3f\") %info)\n",
    "    nowtime = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
    "    print('\\n' + '=========='* 8 + '%s' %nowtime)\n",
    "    \n",
    "print('Finished Training')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T13:33:59.425515Z",
     "start_time": "2020-12-28T13:33:59.407519Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>loss</th>\n",
       "      <th>auc</th>\n",
       "      <th>val_loss</th>\n",
       "      <th>val_auc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9.540213</td>\n",
       "      <td>0.523233</td>\n",
       "      <td>9.009163</td>\n",
       "      <td>0.507570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>9.282163</td>\n",
       "      <td>0.526319</td>\n",
       "      <td>9.060247</td>\n",
       "      <td>0.511278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>9.133593</td>\n",
       "      <td>0.536608</td>\n",
       "      <td>8.906821</td>\n",
       "      <td>0.502245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>8.736319</td>\n",
       "      <td>0.538932</td>\n",
       "      <td>8.213276</td>\n",
       "      <td>0.519633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8.369133</td>\n",
       "      <td>0.544744</td>\n",
       "      <td>8.475071</td>\n",
       "      <td>0.513041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8.118504</td>\n",
       "      <td>0.551813</td>\n",
       "      <td>8.692694</td>\n",
       "      <td>0.512365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.686995</td>\n",
       "      <td>0.559464</td>\n",
       "      <td>7.955071</td>\n",
       "      <td>0.540900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.576513</td>\n",
       "      <td>0.565677</td>\n",
       "      <td>8.241049</td>\n",
       "      <td>0.575489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>9.0</td>\n",
       "      <td>7.217243</td>\n",
       "      <td>0.568084</td>\n",
       "      <td>6.732627</td>\n",
       "      <td>0.521589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>10.0</td>\n",
       "      <td>7.296572</td>\n",
       "      <td>0.577500</td>\n",
       "      <td>5.847867</td>\n",
       "      <td>0.507098</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   epoch      loss       auc  val_loss   val_auc\n",
       "0    1.0  9.540213  0.523233  9.009163  0.507570\n",
       "1    2.0  9.282163  0.526319  9.060247  0.511278\n",
       "2    3.0  9.133593  0.536608  8.906821  0.502245\n",
       "3    4.0  8.736319  0.538932  8.213276  0.519633\n",
       "4    5.0  8.369133  0.544744  8.475071  0.513041\n",
       "5    6.0  8.118504  0.551813  8.692694  0.512365\n",
       "6    7.0  7.686995  0.559464  7.955071  0.540900\n",
       "7    8.0  7.576513  0.565677  8.241049  0.575489\n",
       "8    9.0  7.217243  0.568084  6.732627  0.521589\n",
       "9   10.0  7.296572  0.577500  5.847867  0.507098"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfhistory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T13:34:13.080349Z",
     "start_time": "2020-12-28T13:34:12.756025Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3xUZfb48c8hIBAEQYogJaAiYEBgQUGxLoqoKHZRRFCKCPbVta26X3/grhV1VRALFhAL9oa6oosVBaWINMUAkRY6SCfn98e5MSFMwiSZmjnv12tembn3zr1nJsmcuc/z3POIquKccy51VYh3AM455+LLE4FzzqU4TwTOOZfiPBE451yK80TgnHMpzhOBc86lOE8ELqJEJE1ENolIk0huG08icoiIRHyctYicJCJZBR7PE5Fjw9m2FMd6WkRuK+3zi9nvMBF5LtL7dbFVMd4BuPgSkU0FHqYD24BdweMrVHVcSfanqruAfSO9bSpQ1RaR2I+IDAAuUdUTCux7QCT27conTwQpTlX//CAOvnEOUNX/FrW9iFRU1Z2xiM05FxveNOSKFZz6vyIi40VkI3CJiBwlIt+KyDoRWSYij4pIpWD7iiKiItI0eDw2WP+hiGwUkW9EpFlJtw3Wnyoi80VkvYj8R0S+EpF+RcQdToxXiMgvIrJWRB4t8Nw0ERkhIqtF5FegezHvzz9E5OVCyx4XkYeC+wNEZE7wen4Nvq0Xta9sETkhuJ8uIi8Gsc0GOoQ47sJgv7NF5MxgeRvgMeDYoNltVYH39p8Fnj84eO2rReQtEWkQznuzNyJyVhDPOhGZJCItCqy7TUSWisgGEZlb4LV2FpEfguUrROT+cI/nIkRV/eY3VBUgCzip0LJhwHbgDOyLQ1XgCKATdkZ5EDAfuCrYviKgQNPg8VhgFdARqAS8Aowtxbb1gI1Az2DdDcAOoF8RryWcGN8G9gOaAmvyXjtwFTAbaATUBibbv0rI4xwEbAKqFdj3SqBj8PiMYBsB/gpsAQ4P1p0EZBXYVzZwQnD/AeBzoBaQAfxcaNsLgAbB7+TiIIYDgnUDgM8LxTkW+Gdwv1sQYzugCvAEMCmc9ybE6x8GPBfcbxXE8dfgd3Rb8L5XAjKBRUD9YNtmwEHB/e+Bi4L71YFO8f5fSLWbnxG4cHypqu+qaq6qblHV71V1iqruVNWFwGjg+GKeP0FVp6rqDmAc9gFU0m17ANNV9e1g3QgsaYQUZoz/UtX1qpqFfejmHesCYISqZqvqauDfxRxnIfATlqAATgbWqerUYP27qrpQzSTgUyBkh3AhFwDDVHWtqi7CvuUXPO6rqros+J28hCXxjmHsF6A38LSqTlfVrcAtwPEi0qjANkW9N8XpBbyjqpOC39G/gRpYQt6JJZ3MoHnxt+C9A0vozUWktqpuVNUpYb4OFyGeCFw4lhR8ICItReR9EVkuIhuAu4E6xTx/eYH7mym+g7iobQ8sGIeqKvYNOqQwYwzrWNg32eK8BFwU3L8YS2B5cfQQkSkiskZE1mHfxot7r/I0KC4GEeknIjOCJph1QMsw9wv2+v7cn6puANYCDQtsU5LfWVH7zcV+Rw1VdR7wN+z3sDJoaqwfbHoZcBgwT0S+E5HTwnwdLkI8EbhwFB46+ST2LfgQVa0B3Ik1fUTTMqypBgAREXb/4CqsLDEuAxoXeLy34a2vACcF36h7YokBEakKTAD+hTXb1AQ+DjOO5UXFICIHASOBK4HawX7nFtjv3oa6LsWam/L2Vx1rgvo9jLhKst8K2O/sdwBVHauqXbBmoTTsfUFV56lqL6z570HgdRGpUsZYXAl4InClUR1YD/whIq2AK2JwzPeAv4jIGSJSEbgWqBulGF8FrhORhiJSG7i5uI1VdQXwJTAGmKeqC4JVlYF9gBxgl4j0ALqWIIbbRKSm2HUWVxVYty/2YZ+D5cQB2BlBnhVAo7zO8RDGA/1F5HARqYx9IH+hqkWeYZUg5jNF5ITg2Ddh/TpTRKSViJwYHG9LcNuFvYA+IlInOINYH7y23DLG4krAE4Erjb8BfbF/8iexb8RRFXzYXgg8BKwGDgZ+xK57iHSMI7G2/FlYR+aEMJ7zEtb5+1KBmNcB1wNvYh2u52EJLRx3YWcmWcCHwAsF9jsTeBT4LtimJVCwXf0TYAGwQkQKNvHkPX8i1kTzZvD8Jli/QZmo6mzsPR+JJanuwJlBf0Fl4D6sX2c5dgbyj+CppwFzxEalPQBcqKrbyxqPC59YU6tzyUVE0rCmiPNU9Yt4x+NcMvMzApc0RKS7iOwXNC/cgY1E+S7OYTmX9DwRuGRyDLAQa17oDpylqkU1DTnnwuRNQ845l+L8jMA551Jc0hWdq1OnjjZt2jTeYTjnXFKZNm3aKlUNOeQ66RJB06ZNmTp1arzDcM65pCIiRV4h701DzjmX4jwROOdcivNE4JxzKS7p+gicc+XPjh07yM7OZuvWrfEOJelVqVKFRo0aUalSUaWm9uSJwDkXd9nZ2VSvXp2mTZtihWVdaagqq1evJjs7m2bNmu39CYGUaBoaNw6aNoUKFeznuBJNx+6ci7atW7dSu3ZtTwJlJCLUrl27xGdW5f6MYNw4GDQINm+2x4sW2WOA3mWut+icixRPApFRmvcxqmcEInKtiPwUTGZ9XYj1J4hNRD49uN0Z6Rhuvz0/CeTZvNmWO+eci+IZgYi0BgYCR2KTn08UkfcLTNqR5wtV7RGtOBYvLnr5zp1QsdyfEznnXPGieUbQCvhWVTer6k7gf8DZUTxeSE2KmGRQFZo1g8cfj208zrmyi3S/37p163jiiSdK/LzTTjuNdevWlfh5/fr1Y8KEcOY7io1oJoKfgONEpLaIpGOzEDUOsd1RwSTcH4pIZqgdicggEZkqIlNzcnJKFMTw4ZCevvuy9HS47jrIzIS1a23Zjh3w6aeQ6xPkOZfQ8vr9Fi2yL3R5/X5lSQZFJYJdu3YV+7wPPviAmjVrlv7ACSJqiUBV5wD3YtPmTQRmYBOJFPQDkKGqbYH/AG8Vsa/RqtpRVTvWrVvcNLV76t0bRo+GjAwQsZ+jR8OIETBxYn5fwbvvwkknQYsW8OCDsHp1iQ7jnIugE07Y85b3OX3rraH7/a691u6vWrXnc/fmlltu4ddff6Vdu3YcccQRnHjiiVx88cW0adMGgLPOOosOHTqQmZnJ6NGj/3xe06ZNWbVqFVlZWbRq1YqBAweSmZlJt27d2LJlS1iv9dNPP6V9+/a0adOGyy+/nG3btv0Z02GHHcbhhx/OjTfeCMBrr71G69atadu2Lccdd1xY+w+LqsbkBtwDDNnLNllAneK26dChg0bDli2qY8eqHnOMKqhWrqzap4/q+vVROZxzroCff/55t8fHH7/n7fHHbZ2I/Y+Guqmq5uTs+dy9+e233zQzM1NVVT/77DNNT0/XhQsX/rl+9erVqqq6efNmzczM1FWrVqmqakZGhubk5Ohvv/2maWlp+uOPP6qq6vnnn68vvvhikcfr27evvvbaa7plyxZt1KiRzps3T1VV+/TpoyNGjNDVq1froYceqrm5uaqqunbtWlVVbd26tWZnZ++2LJTC76eqKjBVi/hcjWpXqYjUU9WVItIEOAc4qtD6+sAKVVURORI7Q4nLd/EqVezsoXdvmDULnnwSpkyBffe19V9/DW3aQPXq8YjOudTy+edFr2vSxJqDCsvIsJ916hT//HAceeSRu12Q9eijj/Lmm28CsGTJEhYsWEDt2rV3e06zZs1o164dAB06dCArK2uvx5k3bx7NmjXj0EMPBaBv3748/vjjXHXVVVSpUoUBAwZw+umn06OHjafp0qUL/fr144ILLuCcc84p24ssINoXlL0uIj8D7wJDVXWtiAwWkcHB+vOAn0RkBvAo0CvIXHHVpg089hh89511Rm3dCj16wIEHwpVXwowZ8Y7QudRVVL/f8OGRO0a1atX+vP/555/z3//+l2+++YYZM2bQvn37kBdsVa5c+c/7aWlp7NxZuCV8T0V93FWsWJHvvvuOc889l7feeovu3bsDMGrUKIYNG8aSJUto164dqyPUhh3VMwJVPTbEslEF7j8GPBbNGMoi77qMypXhww9h1Ch47jn7edRRcP/90KVLXEN0LuXkXQh6++02DLxJE0sCZblAtHr16mzcuDHkuvXr11OrVi3S09OZO3cu3377bekPVEjLli3Jysril19+4ZBDDuHFF1/k+OOPZ9OmTWzevJnTTjuNzp07c8ghhwDw66+/0qlTJzp16sS7777LkiVL9jgzKQ0fRR8GEejUyW4PPgjPP2/JIO8ahKwsG3XUvHlcw3QuZeQ140ZK7dq16dKlC61bt6Zq1aoccMABf67r3r07o0aN4vDDD6dFixZ07tw5YsetUqUKY8aM4fzzz2fnzp0cccQRDB48mDVr1tCzZ0+2bt2KqjJixAgAbrrpJhYsWICq0rVrV9q2bRuROJJu8vqOHTtqIsxQlve2icAVV9hIpJNOsqajM86AEhT+cy7lzZkzh1atWsU7jHIj1PspItNUtWOo7VOi6Fw0iOQ3Hf3znzBsGMyfD+eea51W998f1/Cccy5snggioEEDa69cuBDeeQfat4clS2ydKnz2mV2o5lVQnUstQ4cOpV27drvdxowZE++w9uB9BBGUlmbNQmeckX+F8pdfwl//CnXrwrp11pcAXgXVuVTweJLUsPEzgiipELyznTrB+PGwYUN+EsjjVVCdc4nAE0GU7bMP9OoF27eHXr9oEZx2GtxzD0yeDGFele6ccxHjTUMxUtTVkDVr2vIPP7TH++xjiSG4iJGtW+2qZ+ecixZPBDEyfPjuM6WBXQ352GPWR7B6tZWx+PJLSwZ5Wre2RHDMMfm3vAJ6zjkXCd40FCNFVUHN6yiuXds6me+9F/7f/7Nlu3bB5ZdD48bWz9Cnj82hEBQiJDcXZs607ZxLKQkwBG/fvEJkIWRlZdG6desYRlM2fkYQQyW9GjItDW67ze7v2gU//WRnDEFlXH7+Gdq2hf32g6OPzj9jOPJIb05y5ZhPRB5xngiSRFqafegXvKK8USMYOxa++MISRMG5FXr0gF9/tWTRpQvsv3/+88aNi2ydFuci6rrrYPr0otd/+y0ENfv/tHkz9O8PTz0V+jnt2sHDDxd72JtvvpmMjAyGDBkCwD//+U9EhMmTJ7N27Vp27NjBsGHD6NmzZ0leDVu3buXKK69k6tSpVKxYkYceeogTTzyR2bNnc9lll7F9+3Zyc3N5/fXXOfDAA7ngggvIzs5m165d3HHHHVx44YUlOl5peCJIYjVr7n6WkdfPcGxQ6u/VV/PPKDIz7WxBBF54wb9MuSRWOAnsbXmYevXqxXXXXfdnInj11VeZOHEi119/PTVq1GDVqlV07tyZM888EylBJ13etQSzZs1i7ty5dOvWjfnz5zNq1CiuvfZaevfuzfbt29m1axcffPABBx54IO+//z5gBe9iwRNBOZLXz5Dnuuvswz/vjCHveobC8q5n8ETgEsJevrnTtGnRExKUYSKC9u3bs3LlSpYuXUpOTg61atWiQYMGXH/99UyePJkKFSrw+++/s2LFCurXrx/2fr/88kuuvvpqwKqNZmRkMH/+fI466iiGDx9OdnY255xzDs2bN6dNmzbceOON3HzzzfTo0YNjj92jgHNUeGdxOVa1qp0d3HYbfPABrFlT9GijxYtjG5tzpRbFCQnOO+88JkyYwCuvvEKvXr0YN24cOTk5TJs2jenTp3PAAQeEnIugOEUV9rz44ot55513qFq1KqeccgqTJk3i0EMPZdq0abRp04Zbb72Vu+++u8yvKRyeCGIpziMd0tKsTyCUypWtnLZzCW9vQ/DKoFevXrz88stMmDCB8847j/Xr11OvXj0qVarEZ599xqJQZyJ7cdxxxzEu+F+fP38+ixcvpkWLFixcuJCDDjqIa665hjPPPJOZM2eydOlS0tPTueSSS7jxxhv54YcfyvyawuFNQ7GSICMdQl3PUKmSDUXNzITvv4fDDotZOM6VTqQnJAhkZmayceNGGjZsSIMGDejduzdnnHEGHTt2pF27drRs2bLE+xwyZAiDBw+mTZs2VKxYkeeee47KlSvzyiuvMHbsWCpVqkT9+vW58847+f7777npppuoUKEClSpVYuTIkRF/jaH4fASRtG0b5OTk31atyr//yCOwadOez2ncOObtMqFGDR17LDz9NPzf/9mXrLVroVatmIblUpjPRxBZJZ2PINqT118LDAQEeEpVHy60XoBHgNOAzUA/VY38uVBpxkuqWs9q4Q/0gvcLPw71QQ/WFJRXjrSwJUugY0c47ji7HXOMzb4dRUV9mcprjvz9d7uiuW9fu7itevWohuOci7OoJQIRaY0lgSOB7cBEEXlfVRcU2OxUoHlw6wSMDH5GTqgmmf79bWb6li2L/4AvXC40T5UqVlc679a8+e6P69TZ/X6tWnDQQaFHOuy3n33SjhwJwXR0ZGbC8cdbYjj2WDjwwIi+JXtTvbolikcfhddftzIYJRw67VxKmDVrFn369NltWeXKlZkyZUqcIiqdqDUNicj5wCmqOiB4fAewTVXvK7DNk8Dnqjo+eDwPOEFVlxW13xI3DRU11KygmjWL/hAP9QFfrVr4x89TOCGBjXTI6+Tatg2mTrUSpJMn23jPvDOMQw7JP2M4/viYFRv69lsLedYsOOcceOWV/HmanYukOXPm0LJlyxKNz3ehqSpz585NmKahn4DhIlIb2II1/xT+BG8ILCnwODtYtlsiEJFBwCCAJkUNeylKUe3vIrB0qQ2+j8UEw3ltMUU1UVWubJcAd+kCt94KO3fa1ZV5ieGtt+DZZ23bxo3zE8Nxx0GLFlFJDJ07w7Rp8NBDlkvzkoCqF71zkVWlShVWr15N7dq1PRmUgaqyevVqqpSwxkxUO4tFpD8wFNgE/AxsUdXrC6x/H/iXqn4ZPP4U+LuqTitqnxE7I8jISK7xkrm5MHt2fmKYPBmWL7d19ertnhjatMmfGSfCfvwRrr4anngCDj88KodwKWjHjh1kZ2eXeIy+21OVKlVo1KgRlQp9wS3ujABVjckNuAcYUmjZk8BFBR7PAxoUt58OHTpoiYwdq5qermpfZO2Wnm7Lk1lurur8+apPP63ap49qRkb+66tZU/WMM1Tvv191yhTV7dt3f+7Ysba9iP0swXvx4YeqdeuqpqWp/v3vqn/8EckX5ZyLFmCqFvX5XNSKSNyAesHPJsBcoFah9acDH2KjijoD3+1tnyVOBKpl+uBLKllZqi++qDpwoGqLFvmJoVo11ZNPVh02TPWOO8qcGFetUr38cntqs2aqH30UxdfknIuI4hJBtJuGvgBqAzuAG1T1UxEZHJyJjAqGjz4GdMeGj16mqsW2+yT0dQSJZvlyKzSU15Q0c2bR25aiqex//4MrroCLLoK77ipbqM656CquacgvKEsla9bYyKdQv3ORoq91KEZewcfKlWHiRMjOtsl0otRF4ZwrpeISgf+7ppL99y+62FBJR2MFKle2G8CLL8LAgXDCCTBnTulCdM7FnieCVBOqciPYPJhl9OKLVqbip59sAp277gIfBOJc4vNEkGoKV25s1AgaNID//Kf4PoQwVKhgF23PnQvnn28lK95+O0JxO+eixvsInF3g1qWLXcT29dfQrFlEdvvtt9Cpk+WbTz+12QJr147Irp1zJeR9BK54TZrARx9Zz2+3brByZUR227mzJYE//rAzhJYtbY7lJPvu4Vy554nAmcMOg/fft9Kjp54aek7LUqpWDT77zOru9ekDp5wCv/4a93l6nHMBTwQu31FHwYQJMGMGnH12mScDL6htW2t1euwxazJq1cpGGC1aZGcIefP0eDJwLvY8EbjdnXYajBkDkybZ1/dduyK267Q0GDrUhpZWrw5btuy+fvNmq8nnnIstTwRuT336wAMPwGuvwTXXRLxRv2FDmwEtlBhP1pZYvK3MxYlXl3eh/e1vsGIF3H8/HHAA3HlnRHffpEnoorA1ati1ByWsopv8EmROa5ea/IzAFe3ee22+yrvuglGjIrrrUNe1paXB+vXQoYONZE0ZCxZYm1nBSYvA28pczPgZgSuaCDz1lE3bOWSI1Sk677yI7LqoeXoaNLDPxYoVrUUqKytilzUklsWL4dVXYfx4+KGYabpTuq3MxYqfEbjiVapkH1hHHWWf3pMmRWzXvXvbB31urv3s3Rv++leraArwwQc2HfTVV1u9vKS3YoUNmzrmGLuy+6ab7DTowQet4ySUUtaAcq4kPBG4vUtPh3fftU/ls86yacpioHNnayZ/4gk49FAYOTKig5hiY+1aeOYZOPlkOPBAy2obNtjpzy+/wHffwQ03WDNc4bYyEfj73+MTt0spnghcePbf364+rlULune3D7Eoq13bksCPP9rsm0OG2OjWhLdpk3X+nnGGdbQPGGCnPLfdZhX5Zs60+wcfnP+cwjWg6te39rHnn9+z78C5CPNaQ65k5s61po0aNeCrr6xRPwZU4fXX7f5558GOHbBsWQK1nGzZAh9+CC+/DO+9Z48bNYJevez2l7/YB3xJvP22Xdh3zjnWPOeTPLgy8FpDLnJatrTG+5UrrRTF+vUxOayIJYC8vupRo6BFCxvV+scfMQlhTzt22If/pZfaN/9zz7Vp2y6/3GaGW7TIht926FDyJADQs6ddz/H663DrrZGP37k8Rc1hmai3Us1Z7CLvo49UK1ZUPf541S1bYn74xYtVL7rI5k1u1Eh1/HjV3NwYHHjnTtVJk1QHDVLdf38LoGZNm8T5449Vd+yI7PFyc1UHD7bjjB4d2X27lEIcJ6+/HpgN/ASMB6oUWt8PyAGmB7cBe9unJ4IEMm6c/QmdfbZ9QMbBF1+otm9vYQwcGKWD5OaqfvON6rXXqjZoYAerVk314otV33lHdevWKB04sGOHavfuqmlpqp98Et1juXIrLokAaAj8BlQNHr8K9Cu0TT/gsZLs1xNBgnn44fxP4Zh8Jd/Tzp2qTz2l+r//2eN161RXrizBDsaOVc3IUBWxn2PH2muZPl315ptVmza111i5siW9V15R3bQpCq+kGOvXq7Zpo1qjhupPP8X22K5ciGciWALsj1249h7QrdA2ngjKg1tvtT+lf/wj3pGoqur116vut5/qiBGq27fvZeOxY1XT0y3+vFulSvnf/NPSVE89VfX55y3DxNOiRar161uyWr48vrG4pBPPpqFrgU1B88+4EOv7AcuAmcAEoHER+xkETAWmNmnSJJrvlSuN3FzV/v3tz+nRR+Mdjf78s+opp1g4rVpZd0aRMjJ2TwJ5t8qVVUeNUs3JiVXY4fn+e9WqVVU7dVLdvDne0bgkEq8zglrAJKAuUAl4C7ik0Da1gcrB/cHApL3t188IEtSOHapnnWXNK+PHxzsazc215vuDD7a/8uHDQ2zw9dehkwDY60hUb7xh8Z13nuquXfGOxiWJ4hJBNIePngT8pqo5qroDeAM4uuAGqrpaVfNmP3kK6BDFeFw0VawIL71k1xhceil88klcwxGx67lmz4Z//zt/2OmKhX+w9T9P2bj+o48uelhnwlygEMLZZ9uw1AkTvCidi4hoJoLFQGcRSRcRAboCcwpuICIFr0Y6s/B6l2SqVoV33rHpx84+G77/Pt4RUbky3HwzHJo7F669ln1bHEiVawaxZnUuuSOfhKef3rO0Q3q6lYBIZDfcYEWZ/v1vew3OlUHUEoGqTsHa/X8AZgXHGi0id4vImcFm14jIbBGZAVyD9Rm4ZFazJkycCHXrWj2IefPiF8uOHXYxVteulpxGjWJL1zMY0Oorai+ZztHPDeK71pfzZd/RZKdlkIuQnZbBl31HJ/4cACLwn/9At25w5ZXw6afxjsglMS8x4aJjwQLo0sXOEr7+uujqmtGwdKmVzx492u43aQKDB0P//lCvHrm58OKLcMstsHw57LMPbN+e//T0dHtqoucCwK7sPuYYWLLE3ufDDot3RC5BFVdiwhOBi55p0+CEE2zaxcmTrWBdtKhaeYcnnoA337SZbbp3z69Ul5a2x1M2boTGjUNXycjIsDpxSWHRIujUyZLut99auQvnCvFaQy4+OnSAt96C+fOt5zYaVTTXr7ca/5mZcOKJ1kRy3XV2RvLhh3bcEEkAoHp1qwgdyqJFVjcuKWRkWJnwFSusPlHSBO4ShScCF11du8LYsdZsceGFkZuDcsYMa+5p2NBq/O+7Lzz3HGRn24iaQw4JazfFDQ5q2BBGjIhMuFF3xBH2Pn/3nU0vmpsb74hcEvFE4KLv/PPtW/t778HAgdaMUxrbtuUPUW3Xzmr1X3CBffjlfQBWrVqiXYaaOzk93Yp9nnKKnTWAnTmMGRPHSqfhOOccuO8+eO01+Mc/4h2NSyZFXWCQqDe/oCyJ3XmnXaz197+X7HlZWaq33aZar549/5BDVB98UHX16oiEFarUUGEvvGCHrlFD9corVX/8MSKHjrzcXKuMCqrPPhvvaFwCIV4lJqJx80SQxHJzVa+4wv7sHnyw+G137VKdOFH1zDNVK1SwW8+eVi8iDlfT5uaqTp6sesklVn0CVI84QnXjxpiHsnfbt6uefLKVCf/003hHUzrhZGdXIp4IXOLYuVP13HPtT2/w4D3/2VetUn3ggfzaEPXqqd5+uxVcSxCrV6s+8ohqv375y55+WnXq1PjFtId161QzM6363s8/xzuakglVCDA93ZNBGRWXCHz4qIu9rVutxMOcQheSp6XZhVI7d1o/wJAh1u5duXJ84gzT1q02L/3atdC+PQwaBBdfbLN5xlVWFnTubJ0e334L9erFOaAwZWTA4sWhlyfNmN7E48NHXWKpUsUmeC9s1y5bN2OGTfV40UUJnwTAQl64EB5/3AbrXHmlTeU8YUKcA2va1Ep+LF8OZ51lGSuRbdtmPfKhkgAUvdyVmScCFx/Z2aGX//EHHH54bGOJgJo17QTmxx9tANPFF9vAJoAvv7RBU+vWxSGwI4+0y6i/+Qb69UvMYaVr1sC//mWJ6/LLoVKl0NslciHAJOeJwMVHUf/USf7PLmJD+p96Kv9ShrfftksdGjSwEWsFAOQAABqfSURBVK5ffVX6EbSlcu65cO+98MorcMcdMTzwXvz2G1x7rf3Ob7sN2ra1qrVjxuw5prdq1cQvBJjMiuo8SNSbdxaXEynWIThtmvWNV69uL7Vbtz23iepAmdxcm040EYaVTpmiev75NhKsUiXVvn1VZ8zYfZuCbwbYXBeuTPBRQy4hpeAQwY0bVZ95Jv+zePt2G/Z/++0xyIsFh5VOmhTBHYdh1y7Vt99WPfZYe3H77ad6yy2qv/++9+eefrrqAQeobtsW/TjLseISgY8aci6OZs6E444LXfgOojBQZv16m5Bn6VLrN2jZMoI7D2HLFuujePBBqzmVkQHXX299AXmXbe/Nhx9a4cCXXrIBBK5UvPqocwls82aoVi30OpEo9O9mZVm10mrVYMoUmzsi0latskqwjz0GOTlWgPCmm6y/omLFku0rNxcOPRTq17eed1cqPnzUuQSWnm5flEOJSt953rDSZcsiP6x0/nwbP9u4Mdx1lyWczz+32eouvLDkSQCgQgUbkvXVVza02EWcJwLnEkCo4ndVq1pR1Z9+isIBO3WyJpuvv4bLLivbaYeqfUiffbY1NT37LFxyCfz8s5XHPv74oueGDtdll9kb8vjjZduPC8kTgXMJoHdvmxUtI8M+MzMyrCVl5UprVbnvPrveLqLOO8/mPH75Zfv2XlK7dtlUoEcfbVeCT55sVU8XL7bxs61aRS7WWrXs4oxx4+wSbhdZRfUiR+IGXA/MBn4CxgNVCq2vDLwC/AJMAZrubZ8+asilkhUrVM8+2wbaHH206oIFET5Abq5q//52gDFjwnvOpk2qjz2metBB9ryDD1Z9/HFbHk0//GDHe+ih6B6nnKKYUUNROyMQkYbYhPQdVbU1kAb0KrRZf2Ctqh4CjADujVY8ziWjevXsS/eLL8Ls2TBsWIQPIAIjR9oEQoMGWXt+UVassAvSmjSBq67KD27ePGvDL6rHO1Lat7ezjyeeSMwrpJNYtJuGKgJVRaQikA4sLbS+J/B8cH8C0FWkrI2JzpUvItbk/tNP8NBDtuyXX4qu0lFilSpZYaTmzeH0021qtgoVrFN53Dhr6x8wwBLA8OHW5v/VVzb89JxzipwKNCqGDrUX/8knsTtmCohaIlDV34EHgMXAMmC9qn5caLOGwJJg+53AeqB24X2JyCARmSoiU3NycqIVsnMJrVEj2H9/uz9wILRubWcKERkBXrOm7XTzZrvGQNUmbu7b1+aDHjcO+ve3b/9vvGHfzOPh3HPtTMQ7jSMqmk1DtbBv/M2AA4FqInJJ4c1CPHWPP2tVHa2qHVW1Y91ojHl2Lsk8/bQlgksvtc/GlSsjsNOHH95z2a5dsN9+sGSJNck0bx6BA5VB5cqWsN57z2oVuYiIZtPQScBvqpqjqjuAN4DCXyOygcYAQfPRfsCaKMbkXLlw8MHwv//ZaKL337ekMH16GXdaVJnnDRugTp0y7jyCrrjCmq5GjYp3JOVGNBPBYqCziKQH7f5dgUIzkfAO0De4fx4wKejdds7tRVqaDTH94Qfr623Roow7TJaKsI0bQ8+e8MwziT/HQpKIZh/BFKwD+AdgVnCs0SJyt4icGWz2DFBbRH4BbgBuiVY8zpVXmZkwfrxdb7Vxo/XllqovNdRVbenpiVn+eehQWL3aSmu7MvNaQ86VIwsWwJlnwty5NqLzvvtKOKpz3Di4/XZrJsobJdS7d9TiLTVVy4D77mszAbm98lpDzqWI5s2tqei666xvt107qyIRtt69rShdbq79TMQkADamdsgQq2HkiaDMPBE4V85UrQojRsBnn8HOnVb1IclO/MNz6aV2RuBDScvME4Fz5dQJJ9h8B2PH2hfo5csjMLIokdSoYcnglVes7LUrNU8EzpVj1avDgQfa/VtvtfmUhw2zM4VyYcgQ2LbNRhC5UgsrEYjItSJSQ8wzIvKDiHSLdnDOuch54AG7+OyOO6BLF+tQTnqZmXbqM3JkFMqzpo5wzwguV9UNQDegLnAZ8O+oReWci7jata3i9MsvW7me9u1tFsikN3SolcN4//14R5K0wk0EeaUgTgPGqOoMQpeHcM4luAsvtAJ2F14IRx5py5K6M7lnTyuU553GpRZuIpgmIh9jieAjEakOeB1Y55JUgwbw3HN2lrBzJ5x8sjWzjxtnRUcLFh9NeJUqWdmJjz+2qTJdiYWbCPpjV/0eoaqbgUpY85BzLsmtW2fJYMAAKza6aFF+8dFBg5IkGQwcaAlh5Mh4R5KUwk0ERwHzVHVdUEH0H1jJaOdckqtTByZNstkgC/e3bt5sFxonvPr1rSd8zBj44494R5N0wk0EI4HNItIW+DuwCHghalE552KqQgU7Mwhl0SKbkOzJJ+0q5fWJ+hXwqqssuKQ4hUks4SaCnUFV0J7AI6r6CFA9emE552KtqCKjVarACy/A4ME27LRmTbj88vz1b7xhF65t3x6bOIt09NHQtq11Gid173fshZsINorIrUAf4H0RScP6CZxz5URRxUefftq+aP/2G7zzDtxzD3Tvbus3brQWmbZtrbhd69bQq1f+sFTVGE4vLGJDSWfOtKk0XdjCTQQXAtuw6wmWY1NM3h+1qJxzMde7N4weDRkZ9pmakWGPe/e2x02bwhln2BXKF1xgz0lPh1mz4KWXbG6Egw6CKVPyJw9bsMAmODvqKOvPffRRq4G0YUPxsZR69NLFF9sBfShpiYRdhlpEDgCOCB5+p6qRmByvxLwMtXOJT9WSR1YWPPSQJYtZs2wKAYA334SzzrLaRy+8AG3a2O2ww2zdoEHWUZ0nPT0/Ke3V9dfDY49ZKe0GDaLx8pJScWWow0oEInIBdgbwOXYh2bHATao6IYJxhsUTgXPJSdUK382aBR062DUM48dbf0PeRGMiNvNaqFpIGRmWWPZqwQI49FD4v/+DO++M5EtIapFIBDOAk/POAkSkLvBfVW0b0UjD4InAufJl1y749Ve72nnWLPjnP0NvJ1KC/obu3W1nWVl2fYGLyMQ0FQo1Ba3e23NFpIWITC9w2yAi1xXa5gQRWV9gG0/fzqWYtDT7An/OOXDXXfbNP5Rq1SAnJ8ydDh0KS5fC229HLM7yLNxEMFFEPhKRfiLSD3gf+KC4J6jqPFVtp6rtgA7AZuDNEJt+kbedqt5dkuCdc+VPqNFLFSvadWLNm8Nbb4Wxk9NOs4zincZhCSsRqOpNwGjgcKAtMFpVby7BcboCv6rqopKH6JxLJaFGLz33nDUdHXsstGhh223eXMzlAmlpNlfB55/D7Nkxijx5xWTyehF5FvhBVR8rtPwE4HUgG1gK3Kiqe/zWRGQQMAigSZMmHRYt8nziXKo7/3xYs8ZGJbUN1Vu5ahU0amS90U88EfP4Ek2p+whEZGPQtl/4tlFE9jIS+M997AOcCbwWYvUPQEbQ6fwfIORJn6qOVtWOqtqxbt264RzWOVeOqdp8NNOn27wKgwbBihWFNqpTx65ue/HFvV+4kOKKTQSqWl1Va4S4VVfVGmEe41TsbKDwrwlV3aCqm4L7HwCVRKROiV+Fcy6l5F1E/MsvcN11VmuueXOrRL2boUNh0ya7WMEVKRZzFl8EjA+1QkTqi4gE948M4lkdg5icc+VArVrWNDR7NvToAX/5iy1fvTroPzjiCLs98YTXHypGVBOBiKQDJwNvFFg2WEQGBw/PA34KrlN4FOilsei0cM6VK4ceamUu6tSxaw26d4fjj4dp07CqpHPmWG0LF1JUE4GqblbV2qq6vsCyUao6Krj/mKpmqmpbVe2sql9HMx7nXPmnanWN5s61k4GBn1zArv3rWNkJF1Ismoaccy5m0tKs83jBAiuE98KrVXh4Y3/07bdhyZJ4h5eQPBE458ql/faDe++1VqF1Fw62U4Unn2TxYu8uKMwTgXOuXDvoIPh/LzZFevRARz/FiUdv4+ij4dtv4x1Z+EpdljtMngicc6nhqquQnJU8e/rrZGXZHAm9eyd+a9G4cdbUtWiRncksWmSPI5kMYnJlcSR59VHnXKnk5kLLllCnDhs/+pp774UHHrBv2dOmQatW8Q4wtKZN7cO/sLDLcgeKu7K4YulCc865JFOhgtUfuv56qv/yI8OGtWfgQHj+ecsPYPWMDjvMNo2lVatg4kT7YC94e/JJm18nlKKWl4Y3DTnnUke/flbaNKhKmpFhc9eI2KQ5nTtDp06Rn/J40ya7zmH4cBvaevLJcMghloTAPtT79IE77oD337dKq0ccAdWrQ5MmofdZ1PLS8ETgnEsdNWvCJZfYp/LatbutqlcPRo2CZcvgmGPgwgvh4YfD66TdudN2ec891n7frZtd5HbPPbZ+yxbrj/jHP+Ddd2HjRujYMX8mzcxMu+5h82Y7/jff2OxtRx4Zuix3erotjxTvI3DOpZaZM61c6YMPwg037LH6jz+s72D4cNixY/d1lSpZKeyKFa3ppmvX/OoV6ek25Wa9epY0mjaFc8+FCy6w9XPm2LLCH+rhGDcObr/dzhyaNLHYwpq/uYAyT1WZSDwROOfK7Nhj7av3/PlFdgg0agS//77n8goVrKZRs2Zw0kl2BgBWAK9BA5tJLRF5Z7FzzhU0dChcdBF89BGcemrITZYuDf1UVfj++z2XH3JIBOOLMe8jcM6lnnPOgQMOKHYqy1h00iYKTwTOudSzzz5wxRXwwQfw228hN4lFJ22i8ETgnEtNgwZZg//IkSFXh5o7efToknfSJgPvLHbOpa7zz4dJkyA7G6pWjXc0UVXqOYudc65cGzoU1qyBl1+OdyRx5YnAOZe6jj/eruZ6/PGUrk3ticA5l7pE7Kxg2jT47rt4RxM3UUsEItJCRKYXuG0QkesKbSMi8qiI/CIiM0XkL9GKxznnQrrkEivqU8xQ0vIuaolAVeepajtVbQd0ADYDbxba7FSgeXAbBITuvnfOuWipXh369oVXXoGVK+MdTVzEqmmoK/Crqhauqt0TeEHNt0BNEWkQo5icc84MGQLbt8Mzz8Q7kriIVSLoBYwPsbwhUHB+oOxg2W5EZJCITBWRqTk5OVEK0TmXslq1sgpyo0bBrl3xjibmop4IRGQf4EzgtVCrQyzbo+teVUerakdV7Vi3bt1Ih+icc9ZpvHgxvPdevCOJuVicEZwK/KCqK0KsywYaF3jcCCii1JNzzkXRGWdA48Yp2Wkci0RwEaGbhQDeAS4NRg91Btar6rIYxOScc7urWNHqD33yCcybF+9oYiqqiUBE0oGTgTcKLBssIoODhx8AC4FfgKeAIdGMxznnijVggM0+88QT8Y4kprzWkHPOFXTJJTaf5O+/w777xjuaiPFaQ845F66hQ2HDhqInKC6HPBE451xBnTtD+/YpVX/IE4FzzhWUV39o1iz44ot4RxMTngicc66wiy6CWrVSZiipJwLnnCssPR0uvxzeeAOWlf8R7Z4InHMulCuvtHITo0fHO5Ko80TgnHOhHHwwtGkDd99tcxs3bVpuRxJVjHcAzjmXkMaNsyuMc3Pt8aJFNuE9lLsZ7P2MwDnnQrn9dti2bfdlmzfb8nLGE4FzzoWyeHHJlicxTwTOORdKkyYlW57EPBE451wow4fbMNLCbr459rFEmScC55wLpXdvGzqakWFXGzdoAGlp8MEH5a70hCcC55wrSu/ekJVlI4eWLoWHHrIZzMpZmWpPBM45F66rr4ZTT4Ubb4TZs+MdTcR4InDOuXCJwJgxUKMGXHwxbN0a74giwhOBc86VxAEHWDKYORNuvTXe0USEJwLnnCup006zZqKHH4aJE+MdTZlFe87imiIyQUTmisgcETmq0PoTRGS9iEwPbndGMx7nnIuYe++F1q2hXz9YuTLe0ZRJtM8IHgEmqmpLoC0wJ8Q2X6hqu+B2d5Tjcc65yKhaFV56Cdatg/79k3pIadQSgYjUAI4DngFQ1e2qui5ax3POuZhr0wbuu8+GlI4cGe9oSi2aZwQHATnAGBH5UUSeFpFqIbY7SkRmiMiHIpIZakciMkhEporI1JycnCiG7JxzJXT11dC9O/ztb0k7pDSaiaAi8BdgpKq2B/4Abim0zQ9Ahqq2Bf4DvBVqR6o6WlU7qmrHunXrRjFk55wrIRF47jmoXj1ph5RGMxFkA9mqOiV4PAFLDH9S1Q2quim4/wFQSUTqRDEm55yLvCQfUhq1RKCqy4ElItIiWNQV+LngNiJSX0QkuH9kEM/qaMXknHNRc/rpcNVVNqT0o4/iHU2JRHuGsquBcSKyD7AQuExEBgOo6ijgPOBKEdkJbAF6qSZx17tzLrXddx98/rkNKZ05E5KkKVuS7XO3Y8eOOnXq1HiH4Zxzoc2aBUccASefDO+8Y30ICUBEpqlqx1Dr/Mpi55yLpDZt7GKzJBpS6onAOeci7ZprkmpIqScC55yLtMJDSrdti3dExfJE4Jxz0ZBEQ0o9ETjnXLScfjoMHQojRsDHH8c7miJ5InDOuWi6/37IzIS+fSFBS+R4InDOuWjKq1K6di1cfnlCVin1ROCcc9F2+OH5Q0pHjYp3NHvwROCcc7GQN6T0hhvg55/3vn0MeSJwzrlYyJv4PgGHlHoicM65WKlfH559FmbMSKghpZ4InHMulnr0SLghpZ4InHMu1u6/Hw47LGGGlHoicM65WKtaFcaPhzVrEmLie08EzjkXD3lDSt99N+5DSj0ROOdcvFxzDZxyStyHlHoicM65eKlQISGqlHoicM65eCo4pPS22+ISQlQTgYjUFJEJIjJXROaIyFGF1ouIPCoiv4jITBH5SzTjcc65hNSjBwwZAg89FJchpdE+I3gEmKiqLYG2wJxC608Fmge3QUByzOvmnHOR9sADcRtSGrVEICI1gOOAZwBUdbuqriu0WU/gBTXfAjVFpEG0YnLOuYQVxyGl0TwjOAjIAcaIyI8i8rSIVCu0TUNgSYHH2cGy3YjIIBGZKiJTcxLg4gvnnIuKOA0pjWYiqAj8BRipqu2BP4BbCm0jIZ63RxpU1dGq2lFVO9atWzfykTrnXKK45hro1s2GlM4p3JoeHdFMBNlAtqpOCR5PwBJD4W0aF3jcCFgaxZiccy6x5Q0p3XdfuOiimAwpjVoiUNXlwBIRaREs6goUvmLiHeDSYPRQZ2C9qi6LVkzOOZcUGjSwktUxGlIa7VFDVwPjRGQm0A64R0QGi8jgYP0HwELgF+ApYEiU43HOueQQwyGlogk4f2ZxOnbsqFOnTo13GM45F31btkCHDvD771Cjhv1s0gSGD4fevUu0KxGZpqodQ63zK4udcy5RVa0KffrAhg2QnW1DShctgkGDYNy4iB3GE4FzziWyJ5/cc9nmzXD77RE7hCcC55xLZIsXl2x5KXgicM65RNakScmWl4InAuecS2TDh0N6+u7L0tNteYR4InDOuUTWuzeMHg0ZGSBiP0ePLvGooeJUjNienHPORUfv3hH94C/Mzwiccy7FeSJwzrkU54nAOedSnCcC55xLcZ4InHMuxSVd0TkRyQEWxTuOMqoDrIp3EAnE34/d+fuRz9+L3ZXl/chQ1ZAzeyVdIigPRGRqUVUAU5G/H7vz9yOfvxe7i9b74U1DzjmX4jwROOdcivNEEB+j4x1AgvH3Y3f+fuTz92J3UXk/vI/AOedSnJ8ROOdcivNE4JxzKc4TQQyJSGMR+UxE5ojIbBG5Nt4xxZuIpInIjyLyXrxjiTcRqSkiE0RkbvA3clS8Y4onEbk++D/5SUTGi0iVeMcUSyLyrIisFJGfCizbX0Q+EZEFwc9akTiWJ4LY2gn8TVVbAZ2BoSJyWJxjirdrgTnxDiJBPAJMVNWWQFtS+H0RkYbANUBHVW0NpAG94htVzD0HdC+07BbgU1VtDnwaPC4zTwQxpKrLVPWH4P5G7B+9YXyjih8RaQScDjwd71jiTURqAMcBzwCo6nZVXRffqOKuIlBVRCoC6cDSOMcTU6o6GVhTaHFP4Png/vPAWZE4lieCOBGRpkB7YEp8I4mrh4G/A7nxDiQBHATkAGOCprKnRaRavIOKF1X9HXgAWAwsA9ar6sfxjSohHKCqy8C+WAL1IrFTTwRxICL7Aq8D16nqhnjHEw8i0gNYqarT4h1LgqgI/AUYqartgT+I0Gl/MgravnsCzYADgWoickl8oyq/PBHEmIhUwpLAOFV9I97xxFEX4EwRyQJeBv4qImPjG1JcZQPZqpp3hjgBSwyp6iTgN1XNUdUdwBvA0XGOKRGsEJEGAMHPlZHYqSeCGBIRwdqA56jqQ/GOJ55U9VZVbaSqTbFOwEmqmrLf+FR1ObBERFoEi7oCP8cxpHhbDHQWkfTg/6YrKdx5XsA7QN/gfl/g7Ujs1Cevj60uQB9glohMD5bdpqofxDEmlziuBsaJyD7AQuCyOMcTN6o6RUQmAD9go+1+JMXKTYjIeOAEoI6IZAN3Af8GXhWR/liyPD8ix/ISE845l9q8acg551KcJwLnnEtxngiccy7FeSJwzrkU54nAOedSnCcC5wIisktEphe4RezKXhFpWrCKpHOJxK8jcC7fFlVtF+8gnIs1PyNwbi9EJEtE7hWR74LbIcHyDBH5VERmBj+bBMsPEJE3RWRGcMsrjZAmIk8FNfY/FpGqwfbXiMjPwX5ejtPLdCnME4Fz+aoWahq6sMC6Dap6JPAYVjWV4P4Lqno4MA54NFj+KPA/VW2L1QuaHSxvDjyuqpnAOuDcYPktQPtgP4Oj9eKcK4pfWexcQEQ2qeq+IZZnAX9V1YVB0cDlqlpbRFYBDVR1R7B8marWEZEcoJGqbiuwj6bAJ8GEIojIzUAlVR0mIhOBTcBbwFuquinKL9W53fgZgXPh0SLuF7VNKNsK3N9Ffh/d6cDjQAdgWjARi3Mx44nAufBcWODnN8H9r8mfPrE38GVw/1PgSvhzTuYaRe1URCoAjVX1M2ySnprAHmclzkWTf/NwLl/VAlVhweYPzhtCWllEpmBfni4Kll0DPCsiN2Gzi+VVC70WGB1UiNyFJYVlRRwzDRgrIvsBAozwKSpdrHkfgXN7EfQRdFTVVfGOxblo8KYh55xLcX5G4JxzKc7PCJxzLsV5InDOuRTnicA551KcJwLnnEtxngiccy7F/X/JTuQwXXUY+gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXhU9fX48fch7KCCiIosCSpuCKJEUDS4K67U4oLigrtWilsXXL7Vn5W2Wm2Ldalg0QhRUFoVrYqiWEHBEpQdQWSR4BbZkZ2c3x9nQoZkJpkkM3MnM+f1PPNk5s69d04mMGfuZzkfUVWcc8658uoFHYBzzrnU5AnCOedcRJ4gnHPOReQJwjnnXESeIJxzzkXkCcI551xEniBc0ohIlohsFJEO8dw3SCJysIjEfay4iJwuIsvCHi8UkbxY9q3Baz0rIvfU9HiXvuoHHYBLXSKyMexhU2ArsDP0+CZVLajO+VR1J9A83vtmAlU9NB7nEZHrgStU9eSwc18fj3O79OMJwkWlqrs+oEPfUK9X1YnR9heR+qq6IxmxOecSz5uYXI2JyEMiMlZEXhKRDcAVInK8iEwTkbUi8q2IPC4iDUL71xcRFZGc0OPRoeffFpENIjJVRDpWd9/Q82eLyCIRWScifxeRj0VkYJS4Y4nxJhFZLCJrROTxsGOzROSvIrJKRL4C+lTy/twnImPKbXtSRP4Sun+9iCwI/T5fhb7dRztXkYicHLrfVERGhWKbB3SP8LpLQuedJyIXhLZ3AZ4A8kLNdz+GvbcPhB1/c+h3XyUir4lIm1jem2q+zxWa5kRkSvjfLPQ6X4R+j7kiclS013IJoqp+81uVN2AZcHq5bQ8B24DzsS8bTYBjgZ7Y1emBwCJgUGj/+oACOaHHo4EfgVygATAWGF2DffcFNgB9Q8/dCWwHBkb5XWKJ8XVgLyAHWF36uwODgHlAO6AV8JH9N4r4OgcCG4FmYef+AcgNPT4/tI8ApwKbga6h504HloWdqwg4OXT/UeBDoCWQDcwvt+8lQJvQ3+TyUAz7hZ67HviwXJyjgQdC988MxdgNaAw8BXwQy3tTzff54PLvGzCl9G8GXAaswJKfAIcA7YP+f5BpN7+CcLU1RVXfUNUSVd2sqtNV9VNV3aGqS4DhwEmVHD9OVQtVdTtQgH0wVXff84CZqvp66Lm/Yskkohhj/KOqrlPVZdiHcelrXQL8VVWLVHUV8KdKXmcJMBdLXABnAGtVtTD0/BuqukTNB8D7QMSO6HIuAR5S1TWquhy7Kgh/3ZdV9dvQ3+RFLLnnxnBegAHAs6o6U1W3AEOAk0SkXdg+0d6b3dTg30K464E/qeqM0PuzSFVXxHisixNPEK62dvtPKyKHich/ROQ7EVkPPAjsU8nx34Xd30TlHdPR9j0gPA5VVewbd0QxxhjTawHLK4kX4EXs2zDYt/ldHfsicp6IfCoiq0VkLfbtvbL3qlSbymIQkYEiMivUtLMWOCzG84L9frvOp6rrgTVA27B9Yvqb1eDfQrj2wFcx7usSxBOEq63yQzyfwb41H6yqewK/w5oIEulbrMkHABERdv9AK682MX6LfXiVqmoY7ljg9NA38L5YwkBEmgDjgD9izT8tgHdjjOO7aDGIyIHA08AtQKvQeb8IO29VQ3K/wZqtSs+3B9aUtTKGuMqr7H3+KXT+pmH77x92fwVwUA1e08WRJwgXb3sA64CfRORw4KYkvOabwDEicr6I1AduA1onKMaXgdtFpK2ItAJ+W9nOqvo91rb+HLBQVb8MPdUIaAgUAztF5DzgtGrEcI+ItBCbJzIo7LnmWBIoxnLl9dgVRKnvgXalncURvARcJyJdRaQRlsAmq2rUK7JKVPY+fxe6XRHq+L+RsMQEPAv8RkSOFtNJRMKToksCTxAu3u4CrsY6jZ/BvkEnVOhD+FLgL8Aq7Jvn59i8jXjH+DTWVzAHmI5dBVTlRazT+cWwmNcCdwCvYh29F2GJLhb3Y1cyy4C3gRfCzjsbeBz4X2ifw4BPw459D/gS+F5EwpuKSo9/B2sKejV0fAesX6Imor7PoWbAG4B7sP6ig8PjVNWXgIdDx6wH/o1dybgkEvs7OZc+RCQLayq5SFUnBx2Pc3WVX0G4tCAifURkr1CzyP8BO7Bv0c65GvIE4dLFicASrLmiD/AzVY3WxOSci4E3MTnnnIvIryCcc85FlDbF+vbZZx/NyckJOgznnKtTZsyY8aOqRhwWntAEISJ9gGFAFjZ9/0/lnh8I/JmySThPqOqzoeceAc7FrnLeA27TStrDcnJyKCwsjPvv4Jxz6UxEolYDSFiCCA01fBKrP1METBeR8ao6v9yuY1V1ULljewEnAF1Dm6ZgNVw+TFS8zjnndpfIPogewOJQMbJtwBjKipZVRbFKkg2xGacNsBmgzjnnkiSRCaItuxcUKyJyfZx+IjJbRMaVTqVX1anAJGwm57fABFVdUP5AEblRRApFpLC4uDj+v4FzzmWwRPZBRCo6Vr4P4Q3gJVXdKiI3A/nAqSJyMHA4ZQXY3hOR3qr60W4nUx2OlRAmNze3Qv/E9u3bKSoqYsuWLbX8VTJb48aNadeuHQ0aRCvf45xLR4lMEEXsXnGyHVb+YJdQPf1SI7DaKwAXAtNUdSOAiLwNHIctzhJ7AEVF7LHHHuTk5GAFPl11qSqrVq2iqKiIjh07Vn2Acy5tJLKJaTrQSUQ6ikhDoD8wPnyH0qUMQy4ASpuRvsYWKakfqjp5UthzMduyZQutWrXy5FALIkKrVq38Ksy5FFRQADk5UK+e/SwoqOqI6knYFYSq7hCRQcAEbJjrSFWdJyIPAoWqOh4YHFovdwdW0XJg6PBx2BKMc7BmqXdU9Y2axOHJofb8PXQu9RQUwI03wqZN9nj5cnsMMKCm9XfLSZtSG7m5uVp+HsSCBQs4/PDDA4oovfh76VxqycmxpFBedjYsWxb7eURkhqpGXJLWS20451wd9PXX1dteE54gwiSiPW/t2rU89dRT1T7unHPOYe3atbUPwDmXdhYuhPpROgg6VLUIbjV4gggpbc9bvhxUy9rzapskoiWInTt3VnrcW2+9RYsWLWr34s65tKIKzz8P3btDw4bQqNHuzzdtCkOHxu/1MipBnHxyxVvpZ/fdd5d19pTatAluu83u//hjxWNjMWTIEL766iu6devGscceyymnnMLll19Oly5dAPjZz35G9+7d6dy5M8OHD991XE5ODj/++CPLli3j8MMP54YbbqBz586ceeaZbN68OerrjRgxgmOPPZajjjqKfv36sSn0Sw0cOJBx48pWx2zevPmu+4888ghdunThqKOOYsiQIbH9Ys4FIdHDdlLY+vVwxRVwzTVw7LF2FfHPf1qfg4j9HD48fh3UgI1zT4db9+7dtbz58+fv9vikkyrennzSnhNRtfxc8aaqWlxc8dhYLF26VDt37qyqqpMmTdKmTZvqkiVLdj2/atUqVVXdtGmTdu7cWX/88UdVVc3Oztbi4mJdunSpZmVl6eeff66qqhdffLGOGjUq6uuVHq+qeu+99+rjjz+uqqpXX321vvLKK7uea9asmaqqvvXWW3r88cfrTz/9tFs85ZV/L51LutGjVZs23f0/Z9Omtj0DrFihuu++qr//veqOHfE7LzaqNOLnatqU+47Fhx9Gf65Dh+gjAgD22afy42PVo0eP3SacPf7447z66qsArFixgi+//JJWrVrtdkzHjh3p1q0bAN27d2dZJUMU5s6dy3333cfatWvZuHEjZ511VqXxTJw4kWuuuYamTZsCsPfee9fk13Iu8e69N/Jl/r33xvlrc+ooKYFXXoGLL4Z27eDLL2HPPZP3+hnVxFSZoUOt/S5cvNvzAJo1a7br/ocffsjEiROZOnUqs2bN4uijj444Ia1RWENjVlYWO3bsiHr+gQMH8sQTTzBnzhzuv//+XeerX78+JSUlgF01btu2bdd9n+fg6oRkDNtJIT/8AOeeC/37w2uv2bZkJgfwBLHLgAHWfhfv9rw99tiDDRs2RHxu3bp1tGzZkqZNm/LFF18wbdq02r0YsGHDBtq0acP27dspCGufzcnJYcaMGQC8/vrrbN++HYAzzzyTkSNH7uqrWL16da1jcC4hog3PieewnRQxcSIcdRRMmgRPPw0XXhhMHBnVxFSVAQPif6XaqlUrTjjhBI488kiaNGnCfvvtt+u5Pn368I9//IOuXbty6KGHctxxx9X69X7/+9/Ts2dPsrOz6dKly67kdMMNN9C3b1969OjBaaedtutKpk+fPsycOZPc3FwaNmzIOeecwx/+8Idax+Fc3A0dCgMHQvgVdCIu8wP22GPw61/DYYfBu+9CaDxLIHwmtYuJv5cucCUl0Lq19TuUNsUOGwaDBwcbV5xNngyjR8Nf/1qx2TsRfCa1c67umzQJVq+G556Dzz6zbeUGdNRV48bBQw/Z/bw8eOaZ5CSHqniCqKNuvfVWunXrttvtueeeCzos5xInPx/22gv69oWuXa3HdvLkoKOqlU2b4KabbJTSW29BaOxIyvA+iDrqySefDDoE55Jnwwb4179spliTJratV686nSDmzoVLL4UFC2DIEHjwQUi1Nbn8CsI5l/rGjbOv21dfXbYtLw/mz4dVq6Ifl6LWr7fwV62CCRPgj39MveQAniCcc3VBfj506gTHH1+2rXdv+zllSjAx1UDpPL8994QXXoBZs+CMM4KNqTKeIJxzqW3pUvjvf+3qIXxS57HHWrW6OtLM9PHHcPjhMGaMPT7/fAgb9Z6SPEE451LbqFGWGK68cvftjRpBjx7wUbWWqk+6nTttqsZJJ1mJ7gMPDDqi2CU0QYhIHxFZKCKLRaRCmVARGSgixSIyM3S7PrT9lLBtM0Vki4j8LJGxAilRKTK8yqpzGU/VmpdOOSXyjOm8PBvyunFj8mOLwTffWBPSfffZSKXPPrOcVlckLEGISBbwJHA2cARwmYgcEWHXsaraLXR7FkBVJ5Vuw9am3gS8m6hYgcQtCOGcq7kpU2DJkt07p8Pl5dlX9DiUqUmEKVPg00+tLPeLL9oo3bokkcNcewCLVXUJgIiMAfoC86t5nouAt1V1U5V7Vub222HmzOjPT5sGW7fuvm3TJrjuOhgxIvIx3brB3/5W6cv+9re/JTs7m1/84hcAPPDAA4gIH330EWvWrGH79u089NBD9O3bt8pfYePGjfTt27fCccuWLeO8885j7ty5ADz66KNs3LiRBx54gMWLF3PzzTdTXFxMVlYWr7zyCgcddFCVr+VcSsjPh+bNoV+/yM/36mVX/JMnw+mnJze2KLZuhenT4cQT4ZJLLIe1aRN0VDWTyCamtsCKsMdFoW3l9ROR2SIyTkTaR3i+P/BSpBcQkRtFpFBECouLi2sXbfnkUNX2GPXv35+xY8fuevzyyy9zzTXX8Oqrr/LZZ58xadIk7rrrLmIpedK4ceNqHzdgwABuvfVWZs2axSeffEKbuvov1WWeTZvg5ZfhoosgrArybvbc06rapUhH9ZdfWs46/XRrXoK6mxwgsVcQkWpIl/80ewN4SVW3isjNQD7WpGQnEGkDdAEmRHoBVR0ODAerxVRpNFV80ycnJ/qCELVYCOLoo4/mhx9+4JtvvqG4uJiWLVvSpk0b7rjjDj766CPq1avHypUr+f7779l///0rPZeqcs8991Q4LpoNGzawcuVKLgyVgmzcuHGNfw/nku7VV22CXLTmpVK9e1vp5W3bbB3OgIwaBb/4hYUwdiwccEBgocRNIq8gioDwK4J2wDfhO6jqKlUt/Yo+Auhe7hyXAK+q6vaERVkqgQtCXHTRRYwbN46xY8fSv39/CgoKKC4uZsaMGcycOZP99tsv4joQ5UU7LnytB2DXudKlEKPLUPn59sWtdL5DNHl5sHkzhMrZJ0P4eJbsbAvxqqvg6KOtJTuGFuM6IZEJYjrQSUQ6ikhDrKlofPgOoSuEUhcAC8qd4zKiNC/FXaIWhMCamcaMGcO4ceO46KKLWLduHfvuuy8NGjRg0qRJLI905RJBtOP2228/fvjhB1atWsXWrVt58803Adhzzz1p164dr4VWG9m6deuudR+cS2lFRbYowlVX2adwZU480X4mqZmp/HiWr7+GqVPh5z+HDz6A9pEayuuohCUIVd0BDMKahxYAL6vqPBF5UEQuCO02WETmicgsYDAwsPR4EcnBrkD+m6gYKxgwAJYts7LCy5bFbXGIzp07s2HDBtq2bUubNm0YMGAAhYWF5ObmUlBQwGGHHRZjeJGPa9CgAb/73e/o2bMn55133m7nGzVqFI8//jhdu3alV69efPfdd3H5nZxLqFGj7NP3qquq3ne//eCQQxKeIFTh22/hzjsrrny6Y4ddwNRPs+p2vh6Ei4m/ly5pVG3K8b77xj4J7vrr4d//hh9/rPqKIwY7d8KiRbB9uxWO3brVmpQq+34lYt8t6xpfD8I5V3f873+wcGHVndPh8vJgzRqYN6/GL5ufb6W3e/aEPfaAI46Ae+6x5xo1soncw4ZFL4+RhiufernvVDRnzhyuLFdWoFGjRnz66acBReRcEj3/vJX0vvji2I8p7ciePDnqGp2lTUQzZ5bddu60KuIAI0fCnDnW0XzzzTbNKTfse/Ujj9jPVq2sDyK8mSkNVz41qpoWt+7du2t58+fP15KSkgrbXfWUlJTo/Pnzgw7DZYLNm1VbtFC9/PKIT48erZqdrSpiP0ePDj1RUqLatq1q//6qqrp9u+rcuapjx5Yde911qpYm7HbggaqXXWaHqqquW1d2vypR46iDgEKN8rma1lcQjRs3ZtWqVbRq1QqRSNMyXFVUlVWrVvkcCpccb7wBa9fCwIEVniodPVT6zb20Gg7AgAFC8WF51B//EWfmKnPmyq45riedZM1C/frZVUG3bmUL0oUr/7gyAwbEbQxLSkvrTurt27dTVFQU0xwDF13jxo1p164dDVJxRROXXs47z9p+li+HrKzdnoo2l7VNG5u1PO2qpzhu1K1c2esr9u91IN262STrww+vcCoXprJO6rS+gmjQoAEdO3YMOgznXCy++w7eeQd+/euIn+hffx39MIBj78yDUTDqxslwdR2qqZ3CfBSTcy41FBRYr3G50Us//WQ/27WLfFjp6KGsrp2hZcuUqcuUDjxBOOeCp2qjl3r2hNBEz//9z0pW9Ohh8wv++McqquHUqwcnnOAJIo48QTjngvf55zB3Llx9NR99BGeeabli8mS49FKrwxdTNZzevW2GWyVFLF3s0roPwjlXR+TnQ8OGTGx1KWeERh098ojNR9hjj7Ldqhw9lJdnP6dMib6GhIuZX0E45wJTUgKvv7KNLc+9CH37cvLP9+bZZ2HpUuurDk8OMTnmGJtkl+LrVNcVniCcc0m3cye89JLNRxh5yds03vAjetXV1K9vizg2aVLDEzdsCMcd5/0QceIJwjmXVBMmWD/05Zfb4ydyn0f32w/pc1Z8XiAvD2bNgvXr43O+DOYJwjmXcJs32wRpsC/5e+1lxVdnf/Aj7Wf9B7niivjVys7Ls7arTz6Jz/kymCcI51zCbNwIjz4KHTvC/ffbtpNPhunT4cILod7Yl6ymdnUqt1bl+OMt2XgzU635KCbnXNytXQt//7stBb96NZx+etmgot3Koj3/vJVPjVKBtUaaNbPOak8QteZXEM65uPv1r+F3v7N5a9OmwXvvRVhaeu5c+OyziIX5ai0vz2baeR22WkloghCRPiKyUEQWi8iQCM8PFJFiEZkZul0f9lwHEXlXRBaIyPzQEqTOuRS0ciXccYfV2QNbaGfmTBg/3ia8RZSfb01Bl10W/4Dy8mwZuOnT43/uDJKwJiYRyQKeBM4AioDpIjJeVeeX23Wsqg6KcIoXgKGq+p6INAfq4GJ+zqW3pUvh4Yfhueds6GqnTlZOu8oamTt2wOjRcO650Lp1/AM78UT7OXly2eQ5V22JvILoASxW1SWqug0YA/SN5UAROQKor6rvAajqRlXdVMVhzrkEKiiwktv16tnPM8+0hPDcc3DttfDll/CLX8R4snfftTKs8eycDteqla0Z6v0QtZLIBNEWWBH2uCi0rbx+IjJbRMaJSPvQtkOAtSLybxH5XET+HLoi2Y2I3CgihSJSWFxcHP/fwDkHlC3Ws3y51dVbvhw+/BDOOAOWLIGnn47hqiFcfr59iJ97bqJCtiuHTz6xSxtXI4lMEJGWcCu/OtEbQI6qdgUmAvmh7fWBPOBXwLHAgcDACidTHa6quaqa2zoRl6nOOQCGDNl9DWaw0akLFkDbSF/7KrNmDbz+us2Ua9gwbjFW0Lu3TZabPTtxr5HmEpkgioD2YY/bAd+E76Cqq1Q1tDAgI4DuYcd+Hmqe2gG8BhyTwFidcxHs2AGPPw5FRZGfj7aIT6XGjrUO5EQ1L5Uq7XvwZqYaS2SCmA50EpGOItIQ6A+MD99BRNqEPbwAWBB2bEsRKb0sOBUo37ntnEuwc8+F226DaEuSly7WUy35+XDkkTZXIZHat7ea4F64r8YSliBC3/wHAROwD/6XVXWeiDwoIheEdhssIvNEZBYwmFAzkqruxJqX3heROVhz1YhExeqcK/PNN9Z8BHDLLVYS49lnq1isJ1YLF9rEiKuvLjdjLkHy8uwKQsu3bruYqGpa3Lp3767OuZrbulX1kUdUmzdXfeyxis+PHq2ana0qYj9Hj67Bi9x9t2q9eqrffFPLaGP0zDOqoLpwYXJerw4CCjXK56qX2nDO8d578Mtf2hf888+Hn/2s4j5VLtZTlZ07YdQo6NMH2rSpev94CO+HOOSQ5LxmGvFSG85luLvvtjkNO3bAm2/a7OcDD0zAC02aZL3die6cDnfYYbDPPt5RXUN+BeFcBtqyxRJC8+Zw3nm2ctudd0bvjI6L/Hxo0QIuuKDqfeNFpKwfwlWbX0E4l2HefBM6d7Z6SWAF9e65J8HJYf16+Ne/4NJLE/xCEeTl2Wy+lSuT+7ppwBOEcxli8WK7Wjj/fGjUCPrGVPgmTsaNs1WDElG5tSo+H6LGPEE4lwFeesmuGv77X1vAZ9YsOO20JAaQn2+dxFFLuyZQt27WluYJoto8QTiXplRtRTeAHj2sqvaiRXDXXdCgQRIDWbLEJqsla+5DefXr2ypzniCqzROEc2lowQIbmVS61MJBB9nibckaXbqbF16wxHDllQG8eEheni1QtGZNcDHUQZ4gnEsj69fbam5du0JhIZx1VsCTiEtKLEGcdpqVvghK7972Rnz8cXAx1EGeIJxLE9On27D/Rx+11pyFC2HQoGBadXaZMsVWFUrm3IdIevSwdjVvZqoWnwfhXB23bZtVzS5dze211+zzMCU8/7x1EF94YbBxNGkCxx7rCaKa/ArCuTpqzRorj3H88TbprUULeOutFEoOP/0Er7wCl1wCzZoFHY31Q0yfXnFhCxeVJwjn6piSEhg5Eg49FJ56yhLE1q1VH5d0r75qw6iCbl4qlZdnmfTTT4OOpM7wBOFcigtfC7pdO2tKuu46m1YwYwY88URqfEGv4PnnbR3SE08MOhJzwgnWIePNTDHzPgjnUljpWtClrSIrV1qiuPlmu3oItAO6MitWwAcfwP33W8CpoEUL6NLFE0Q1pMhfzjkXyd13V2wyLymBt99O4eQAVtZbFa66KuhIdte7N0ydak1NrkqeIJxLQSUldvWwYkXk52u0FnSyqFppjd69rYkpleTlWef5558HHUmdkNAEISJ9RGShiCwWkSERnh8oIsUiMjN0uz7suZ1h28eXP9a5dLVunY3IvOKK6CUxarQWdLJMm2Y1PYIozFcVL9xXLQlLECKSBTwJnA0cAVwmIkdE2HWsqnYL3Z4N2745bHsSC8g7F4wff7Sfe+0FRx9trTQjR8ZpLehkys+3IC+6KOhIKmrTxuqOfPRR0JHUCYm8gugBLFbVJaq6DRgDJLPAsHN1wsqVNiopOxuWL7dtzz5rVxBXXAHDh9tzIvZz+PBaLv2ZSFu2wJgx8POf2ypEqSgvz2Z4l5QEHUnKS2SCaAuEt6AWhbaV109EZovIOBEJL9bSWEQKRWSaiERYIRdE5MbQPoXFxcVxDN25xFu/Hu67z4atjhoFN90U+TN1wABYtsw+z5YtS+HkAPD669ZGlipzHyLJy4NVq+CLL4KOJOUlMkFEGmNRvmzYG0COqnYFJgL5Yc91UNVc4HLgbyJyUIWTqQ5X1VxVzW3dunW84nYu4X76yeomDR1qC/d88QX85S+w995BR1ZL+flWlO+UU4KOJLreve2n90NUKZEJoggIvyJoB3wTvoOqrlLV0jmgI4DuYc99E/q5BPgQODqBsTqXcKplk3ibNYPf/tYev/QSHHhgsLHFxbffwoQJVtY7KyvoaKI76CDYf39PEDFIZIKYDnQSkY4i0hDoD+w2GklEwqvTXwAsCG1vKSKNQvf3AU4A5icwVucSaupUa9k47jgrBwRw220pVDcpHgoKrB0s1eY+lCdifwxPEFVKWIJQ1R3AIGAC9sH/sqrOE5EHRaR0VNJgEZknIrOAwcDA0PbDgcLQ9knAn1TVE4SrcxYvhosvhl697P4zz9gIpbSjaqU1jj/eikSlurw8m0xSOirARZTQUhuq+hbwVrltvwu7fzdwd4TjPgG6JDI25xJtyxb7vNy8GR54wJb6bN486KgS5LPPYN48+Mc/go4kNuHzIbKzg40lhflMaufiaNMmGDHCWloaN4bRo+3K4f770zg5gHVON2oEl14adCSx6dIF9tzTm5mq4AnCuTjYudNaWA45xIrrlc7DOuss6w9Na9u2wYsv2nCsFi2CjiY2WVlW3dUTRKU8QThXSxMmwDHHwDXXwAEHwIcfwsknBx1VEv3nPzavIBVLa1Smd29YsKBsCrurwBOEc7WwfTvccgts2GATiKdNg5NOCjqqJMvPt8ukM84IOpLqKe2HmDIl2DhSmCcI56ppxQq4/XbrfG7QwEpvL1hgze+psvRB0hQX2xXEFVdA/Tq2vExurvWbeDNTVJn2z9m5mIWv5JaTY/WRhgyx0hj/+IddLYCN6mzUKMhIA/Tii7a2QiqX1oimUSPo2dML91XCE4RzEcChLKQAACAASURBVJSu5LZ8uQ3xX74cbrgBHn4YLrkEFi5M7WoSSZOfD927w5FHBh1JzeTl2doQGzcGHUlK8gThXAT33ltxJTewatEvvOBD5wGYPds+XOvi1UOpvDwbgjZ1atCRpCRPEM6Vs2FD9Am2332X3FhSWn6+dcJcdlnQkdTc8cdbG6L3Q0TkCcK5kKIi+M1voF276Puk9EpuybRjh7XDnXce7LNP0NHU3J57Wu0TTxAR1bFhB84lxqpVcPDBNmz14ouhc2f40592b2ZK+ZXckmnCBPj++7rdvFQqL89GHWzbBg0bBh1NSvErCJeRVOGdd6xGEkCrVvD00/DVVzaf4f/+r46t5JZszz9vVw5nnx10JLWXl2eFs2bMCDqSlOMJwmWUrVvhueega1f7bPvnP63PAWwmdE5O2b51aiW3ZFq9GsaPtzckHb5xn3ii/fThrhV4gnAZY8oUSwDXXmv9kvn5dsWQqksnp6yxY605Jh2alwD23dcms3g/RAWeIFxaW7LEKlGDLfGZmwvvvgszZ9q6NunwBTjp8vOtGmq3bkFHEj95efDxx3a56HaJKUGIyHEiskfY4z1EpGfiwnKudqZNg4suslnPt99u2/bZB954w0oGSaQV013lCgqgbVtbJ7WoyGZRp4vevWHtWpg7N+hIUkqsVxBPA+FTDX8KbXMupUycaFWcjz8e3n/fhq2OGRN0VGmgdGr5N6Fl5dessccFBcHGFS/hCwi5XWJNEKKqWvpAVUvwIbIuRWzaZINQAL78Er79FoYNs6J6f/yjleB2tRRpavmmTbY9HWRn2wQYTxC7iTVBLBGRwSLSIHS7DVhS1UEi0kdEForIYhEZEuH5gSJSLCIzQ7fryz2/p4isFJEnYozTZZDvv4ff/c4mrz33nG277jpYtAgGD07zFdyS7euvq7e9rhGxq4jJk20MtANiTxA3A72AlUAR0BO4sbIDRCQLeBI4GzgCuExEjoiw61hV7Ra6PVvuud8D/40xRpchFiywwnnZ2fDQQzZKsXt3e65hw7pXdbpOiDaFPJ2mluflWRPakiq/+2aMmBKEqv6gqv1VdV9V3U9VL1fVH6o4rAewWFWXqOo2YAzQN9bARKQ7sB/wbqzHuMxw3XW21vM118AXX8Brr0GPHkFHleauu67itnSbWu79EBXEOorpOREZWf5WxWFtgRVhj4tC28rrJyKzRWSciLQPvV494DHg11XEdaOIFIpIYXFxcSy/iqsDwtdhyM6GW2+1jufSP/GIEday8fTTtga0S4J586BJE2jfPn2nlh9xBLRs6QkiTKwX42+G3W8MXAh8U8UxkQYSlm/cewN4SVW3isjNQD5wKvAL4C1VXSGVjEdU1eHAcIDc3FxvOEwDpYNlSvtDv/4annrKymx//TW0bm11klwSLVsGr7wCd90FjzwSdDSJU69eWT+EA2JMEKr6r/DHIvISMLGKw4qA9mGP21EuqajqqrCHI4CHQ/ePB/JE5BdAc6ChiGxU1Qod3a5u27wZ5s+HWbNsQmu0dRgaNCjrZ3BJNmyYfXgOHhx0JImXl2dlRL77ztbZznA17c7rBFTVOzUd6CQiHbHO7f7A5eE7iEgbVf029PACYAGAqg4I22cgkOvJoW5TtXlILVva45tustI3ixaVTV7t2zf6oJgVKyJvdwm2dq2ttdq/f+V10NNFaT/ElCk20zLDxZQgRGQDZc1DCnwP/KayY1R1h4gMAiYAWcBIVZ0nIg8Chao6HhgsIhcAO4DVwMAa/RYu5cyebRNuZ8+2q4PZs+HAA8vKXmzYYOVvLr4YjjrKiucdeCAcdFDkxXrSabBMnTJ8uC3HedddQUeSHMccY53vH33kCQKbABfbjiJ7Y1cOjUObVFVTpvxhbm6uFhYWBh1GRimtcjp7tt2WLLEq0GBfOMeOtbkIXbvaLTc38mCYcOX7IMD+v6Zbf2idsG0bdOwIhx9uU9QzxWmnWcXazz8POpKkEJEZqpob8UlVrfIGXA/MAdYAk4DNwAexHJusW/fu3dXVzujRqtnZqiL2c/TosufWrVOdMkV161Z7PGyY6h57qFrjkR3TqZPq+vX2/KJFqkuWqO7cGd84XBK98IL9cd9+O+hIkuv+++0f39q1QUeSFFiLTsTP1ZiuIERkDnAsME1Vu4nIYcD/U9VL45LC4sCvIGon0jf3Bg2saOeaNbB0qW37/HMr4jlhArz5ZtnVwZFHQrNmwcTuEkDV/tA7d8KcOZlV3fD99+H00+Gtt9JjQaQqVHYFEWsn9RZV3SIiiEgjVf1CRA6NY4wuYJFGD23fbk1H/frB9deX9RMAnHWW3Vyaev99++OPHJlZyQHguONsOv7kyRmRICoTa4IoEpEWwGvAeyKyhqrnQbg6JNrooZ07vRpqRnr0URvmefnlVe+bbpo1szHVPh8i5nkQF4buPiAik4C9gHcSFpVLqncq+Uv66KEMNHeutSEOHQqNGgUdTTDy8uDxx61McOPGVe+fpqq9opyq/ldVx6vVV3J1mCr86U9wzjk2xL1Jk92fT7dSOy5Gjz1mf/ybbw46kuDk5dkorv/9L+hIAuVLjmawtWutjEX//lYhdcQIK7GTrqV2XAy+/dZGLFx7Ley9d9DRBOeEE+xnhjczeWHkDLR8uS2i07KlTWbbf39LCgMGeELIeH//O+zYUbZOa6Zq1cqKfmV4gvAriAzzzjs2evH+++1xmzaZN0jFRbFxI/zjH/Dzn9uU9kzXuzd88omN1MhQniAyhKotv3nOOdZ8dMMNQUfkUs5zz9mkl0wpq1GVvDyrCTNrVtCRBMYTRAbYuNFqHt1zj/U3fPKJVVBwbpedO+Gvf4VeveD444OOJjX4AkKeIDLB0qXw7rs2tL2gwAaoOLebV1+1fyh+9VCmXTtbuSqDE4R3UqexefOsn61LFyukt88+QUfkUpKqfXs46CCrue7K5OVZx51qRnbW+RVEGlK1+QtdusC4cbbNk4OL6pNPbDjbHXdAVlbQ0aSWvDxb63bRoqAjCYQniDSzYYOVsb/vPrjsMuuUdq5Sjz1mcx4GDgw6ktST4f0QniDSyJdfWp2x116z//OjR3t/g6vCl1/aP5hbbvFyvJEceqgthJ6hCcL7INLIrFnwww/w3ntw6qlBR+PqhL/9zeq6DxoUdCSpScSuIjI0QST0CkJE+ojIQhFZLCIV1pQWkYEiUiwiM0O360Pbs0VkRmjbPBHJ4KIwlSspKVvG86KLYPFiTw4uRqtW2dyHK66w6fQusrw8G+G1cmXQkSRdwhKEiGQBTwJnA0cAl4nIERF2Hauq3UK3Z0PbvgV6qWo3oCcwREQOSFSsdVVpf8Nxx8HChbZtr72CjcnVIU8/DZs3w513Bh1JasvgfohEXkH0ABar6pJQ5dcxQExj6FR1m6puDT1shPeVVLBoEfTsCePHw8MPwyGHBB2Rq1O2bLG6S2efbWOhXXRHHWWLq3/0UdCRJF0iP3jbAivCHheFtpXXT0Rmi8g4EWlfulFE2ovI7NA5HlbVCgsUiciNIlIoIoXFxcXxjj9l/ec/0KOH9Te8+66NTszAIdquNgoK7B/Qr34VdCSpr359m2HuVxBxFekjq/wC2G8AOaraFZgI5O/aUXVFaPvBwNUisl+Fk6kOV9VcVc1t3bp1HENPbdOm2dKfM2Z4f4OrgZISG+bWrRucckrQ0dQNvXvbQkqrVwcdSVIlMkEUAe3DHrej3DKlqroqrClpBNC9/ElCVw7zgLwExVknrF9fVjPs//0/+PhjK7rnXLW9844tAPKrX/mlZ6xK+yE+/jjYOJIskQliOtBJRDqKSEOgPzA+fAcRaRP28AJgQWh7OxFpErrfEjgBWJjAWFPawoXW33DOOdanWK9exdXfnIvZo49anaFLLgk6krqjRw9o2DDjmpkSNg9CVXeIyCBgApAFjFTVeSLyIFCoquOBwSJyAbADWA0MDB1+OPCYiCjWVPWoqs5JVKyp7M03bRGfhg3h5Zc9Mbha+uwzmDQJ/vxnm//gYtO4MRx7bMYlCFEt3y1QN+Xm5mphYWHQYcRNSQk89JAt7HPMMVZss0OHoKNydd6AAfDGG7BihY+Jrq6777arr7Vr02rWuYjMUNXcSM/58NEU9tlncOWVMGWKJwcXBytWwNixtlqUJ4fqy8uz5Vg//TToSJLGE0QKKCiwsvP16kHbtjBsmN0fOxby871ZycXJsGH287bbgo2jrurVyzr1M6iZyWsxBaygAG68ETZtssfffGMTW/fZx1oDnIuLdetg+HDrmPbL0Zpp0cImzWVQgvAriIDddVdZcihVUgL33htMPC5NPfus1WbxFeNqJy8Ppk6F7duDjiQpPEEkSUkJzJkDTz5p60IvXmzbv/8+8v5ff5282Fya277dmpdOPhm6V5hq5KojL8++0X3+edCRJIUniARbvBjOP9+ajLp2tarKH39clgCiXe17K4CLm1desQ5qv3qovQwr3OcJIk5++gnefx8eeMDKX/z977Z9r71sTZZ+/azDeelSSw6lJTL+8IeKi/o0bWpLhjpXa6pWVuOww3x5wXjYf384+OCMKdznndQ1tH27zTMqKbEr96lTbQRcvXrWj1U68qh1a/jii+jnKe2IvvdeSxwdOlhy8A5qFxcffmjjpYcPt3+crvby8uD11+0/f5q/p54gYrRypX1pmDzZbi1b2uN69eDII+HEE62eV69esOee1Tv3gAGeEFyCPPYY7LuvTahx8ZGXZwstLViQ9qXS0zv9xSB8DkJOjj1WhWXLyva59lorXXP55TBqFBxwgPUrlHrqKWsq6tOn+snBuYRZsMBqw996q5WKcPHRu7f9zIB+iIwutVF+DgJAVpbNol+/Hn78EVq1sjIXy5bZF4du3aw8vHMp74YbYPRo66DeZ5+go0kfqjaj9ZRT7EOkjqus1EZGf9Tde2/FOQg7d9rtmWfKapldeGHyY3OuVr7/Hl54wS5/PTnEl4h9W8yAK4iMbmKKNtdg0ya7svDmIldnPfmkjaS4446gI0lPeXl2ZbZ8edCRJFRGJwifg+DS0qZN1jF2wQW+WHmibNhgP3Nyyjov01BGJ4ihQ30OgktD+fmwapWvN50oBQVWi7/U8uXW5JCGSSKjO6nB/qY+B8GljZ07bVLc3nvb4uW+pGj85eREblrKzt59+GMd4Z3UlfA5CC6tvPGG1Xd5+WVPDokSrfMyDQuoJbSJSUT6iMhCEVksIkMiPD9QRIpFZGbodn1oezcRmSoi80Rktohcmsg4nUsbjz5q33B96F3iZFDnZcIShIhkAU8CZwNHAJeJyBERdh2rqt1Ct2dD2zYBV6lqZ6AP8DcRaZGoWF2KiTR70VVt2jSrBHnHHT5ZJ5EidV4CnHtu8mNJsEReQfQAFqvqElXdBowB+sZyoKouUtUvQ/e/AX4AWicsUpc6SmcvLl9uE5LSuAMw7h57zBa1ufbaoCNJbwMGWG2r7GxrxuvQwfp9Roywip1pJJEJoi2wIuxxUWhbef1CzUjjRKR9+SdFpAfQEPgqwnM3ikihiBQWFxfHK24XpEizFzdt8hWUqrJkCfz733DzzdC8edDRpL8BA6xDuqTEvsRMnWpDin/+c5g9O+jo4iaRCSJSD1n5IVNvADmq2hWYCOTvdgKRNsAo4BpVLalwMtXhqpqrqrmtW/sFRlrIoA7AuPrb36xOzC9/GXQkmalFC3j7bdhjDzj77LT595rIBFEEhF8RtAO+Cd9BVVep6tbQwxHAruWuRGRP4D/Afao6LYFxulTSvsJFpMnKskqJO3cmN566YPVqGDnSqkkecEDQ0WSu9u0tSfz0k1XuXLMm6IhqLZEJYjrQSUQ6ikhDoD8wPnyH0BVCqQuABaHtDYFXgRdU9ZUExuhSTY8eFbc1amQffFddZbXVX37ZLu2deeYZ+1C6886gI3FdusBrr8FXX0HfvrBlS9AR1UrCEoSq7gAGAROwD/6XVXWeiDwoIheEdhscGso6CxgMDAxtvwToDQwMGwLbLVGxuhTx0UfWjn788dbxJ2Idgf/8py3FN26cXUlceikcfbT9R0yTiZ41tnWrLV945pm2pq0L3sknW6HEyZNtHY66/GVGVdPi1r17d3V1WHGxatu2qgcfrLp+ffT9duxQLShQ7dRJFVS7d1f9z39US0qSF2sqee45ex8mTAg6ElfeY4/Z32bw4JT+9wkUapTP1YyuxeRShCoMHAjFxdZ8tMce0ffNyrK29vnzbVWv1att/HmvXjBxYmZdUZSuN92lC5xxRtDRuPLuvNPmpDz+uP2d6iBPEC54f/2rrXz22GPWdBSL+vUtqXzxhbXBFxXZh+TJJ2fMgvK8+y7MnQt33eVlNVLVo4/CJZfAr38NL70UdDTV5gnCBet//4Pf/tZKQ9x6a/WPb9jQJtJ9+aV9U1u0CE46yZLFtDQf/PbYY9Z5f9llQUfioqlXz6rr9u4NV18NH3wQdETV4gnCBWftWujf35Zv/Oc/a/ctuHFjmwPw1Vf2rW3mTOvsPvdcmDEjfjGnilmz4L337Hdu2DDoaFxlGje2ARWHHGJfhOrQRDpPEC4YqrZm8ooVMGYMtGwZn/M2bWpNLkuXwh/+YDNcc3PtP+acOfF5jVTwl7/Y4uk33RR0JC4WLVvWyYl0niBcMJ55xoatDh0Kxx0X//M3bw53322J4oEH7NL+qKPsiuWLL+L/esm0ciW8+CJcd138EqtLvNKJdBs3WpKoAxPpPEG45Js1C26/3WabJnrVs732gvvvt0Rx993w5pvQubNNulu8OLGvnSh//7uNrb/99qAjcdVVOpFu8WL42c9SfiKdJwiXXBs32qiOvfe2yUT1kvRPcO+97Wpl6VIbfjhunFXgvP76urXw/IYN8I9/QL9+0LFj0NG4mjjlFOu4/ugj+6KSwhPpPEG45Lr1Vvv29OKLEESBxdat4c9/ts7sW2+1+k6dOsEvfmFNN6lu5EhYt876WVzd1b+/DaZ45RX7wpKi83c8Qbjkyc+3q4b/+z+brxCkNm1g2DBLVtdea7X8DzrImm2++y7Y2KLZscPmjJx4IvTsGXQ0rrbuvNP+vQ0bZoMOUpAnCJccX3xh39JPPtkSRKpo396abBYtshnaTzwBBx4Iv/mNbU+lle3+/W9rDkt0v41LDhGby3LxxfY3HTMm6Igq8AThEm/zZut3aNrUPmSzsoKOqKKOHa35ZsECW/Tlz3+GW27ZfWW7G26wJqlkKyiwooWXXmozyDdsSH4MLjHq1bOr6t69rT9i0qSgI9qNaIq2fVVXbm6uFhYWBh2Gi+SWW+zb+Ntv28iluuCAA+DbbyM/17SpjWffc8/Kf8ayT1XJsnQJ1vBV9po2tSUvBwyI3+/rgrVmjTUdFhXBlCk22ilJRGSGquZGfM4ThEuoV16xq4ff/AYefjjoaGJXr170jsM777Rv8Rs2wPr1u/8svb91a+RjyytNNtGSyIsv2vnKy862JS9d+lixwuYEidgEz2iLZ8WZJ4hUV1Bgay5//bWtgzB0aHp8O1yyxIrvHXGEDelr0CDoiGKXkxN5+GusH8zbtlVMIpESSlXPRVtrXSSlh0e6Gpo9G/LyLDlMmWJLmSZYZQmifsJf3VWufBPC8uX2GOp2kti2zdrM69WzKpZ1KTmAJelITTtDh8Z2fMOG0KqV3WojWqLq0KF253WpqWtXm0h31lk2ke6dd6yWU0C8kzpo9967+4cQ2ON77w0mnngZMgQKC60IX05O0NFU34AB1s6fnV22sl0Q7f5Dh1piCledROXqntKJdP/9r1WADfJKMdpKQvG4AX2AhcBiYEiE5wcCxcDM0O36sOfeAdYCb8byWnVqRbmSEtX588tWnIp0Ewk6ypobP95+h0GDgo4kPYwerZqdbf8msrPtsUt/f/6z/T+6446EvgyVrCiXsD4IEckCFgFnAEXAdOAyVZ0fts9AIFdVB0U4/jSgKXCTqp5X1eulfB/ETz/ZELa334a33iprx27QALZvr7h/s2Y2iqay1dVS0YoV0K2bNYFMnRro5bFzdZqqrUg3bJjNl7jzzoS8TGV9EIlsYuoBLFbVJaq6DRgD9I31YFV9H6jbA75LF7Hp08faos8/3y4du3a1YZ/Ll9uymeWbEOrXt4TSrRt88kkwsdfEjh022WzbNhg71pODc7UhYjOsL7rISqsEMJEukZ3UbYEVYY+LgEj1AfqJSG/sauMOVV0RYZ+6YcsWazd86y27lVYLPfRQm0V8zjk2QqFRo7JjStu0y49iys6GK6+0/e+912Yfp3pH7wMP2MiLggJbHMU5Vzv16tnkzO+/t/6I/fdPbpmaaG1Ptb0BFwPPhj2+Evh7uX1aAY1C928GPij3/MlU0gcB3AgUAoUdOnSIb8NcrJYsUX3ySdVzz1Vt0sTaDJs0UT3nHNUnnlD96quan3vdOtWrr7ZzHnus6sKFcQs77t57z9rIr7026EicSz+rV6secYTqXnupzp4d11MTUB/E8cADqnpW6PHdoYT0xyj7ZwGrVXWvsG0nA7/SVOqD2LrVviWXXiWULj5z0EF2hXDOObYmcpMm8XvNceNs5bAtW6wt8qabUmuR+u+/t8V4WrWyNaabNQs6IufSz9df2zK6cZ5IF9Q8iOlAJxHpCKwE+gOXlwusjaqW1jO4AFiQwHhqbsWKss7liROtf6BhQ7vUu/lmSwqdOiXu9S+6CHr1goEDrWzFm2/a8NH99kvca8aqpASuuMJKUE+c6MnBuUTp0ME+h/LybEW6ZEyki3ZpEY8bcA7Wt/AVcG9o24PABaH7fwTmAbOAScBhYcdOxobAbsb6L86q7LVqPMw10hDCbdtUP/xQ9Te/UT3yyLKhp9nZqrfcovrGG6obN9bs9Wpj507VYcNUGzVSbd1a9fXXkx9DeUOH2nszYkTQkTiXGd5/X7VBA9WTTlLdsqXWp6OSJqaEJohk3mqUIEaPVm3atCwBgGpWlmrjxna/QQPVU09VffRR1XnzbP5CKpg7V7VbN4vxxhuDSVaqqpMn2/vVv3/qvDfOZYKCAvv/37OnaocOtZojU1mCyOxaTNHKGDRvbiV4TzvNCqaloq1b4Xe/s7LUBx8Mo0dDjx7Je/1Vq2wYbqNG8Nlnqfs+OZeuLrus4tDXGlT6DWoeROr7+uvI23/6CS68MLU/9Bo1suqokyZZsujVCx580OYiJJqq9Yd8/73Nd0jl98m5dBVpjlScy/RkdoKIVvCsLhVCO+kkmDXL1ri9/37rwCqdf5Eow4ZZR/mjj0L37ol9LedcZCuiTBmL9sW3BjI7QaRLIbQWLayJ6aWXbNhtt27w7LOJWQi9sNDWdujbF375y/if3zkXmyR8wc3sBJEqFTvjpX9/qyffs6ctj3nhhdHXE6iJdeushHebNrY8ZyrNxXAu0yThC25mJwiwZLBsmY3nX7as7iaHUu3bw3vv2YS6t9+2pQvfeqv251W19RGWL7crlb33rv05nXM1l4QvuJ4g0lG9elb5cfp02HdfOPdcuPXWiutOVMeIEfDyy/DQQ9Yh7pwLXoK/4HqCSGddu1rpizvvhKeegmOOgRkzqn+eOXPgttvgzDOt/8E5lxE8QaS7xo2tuWniRNi40RZF/8MfYOfO2I7/6Se45BLrCB81yq5OnHMZwf+3Z4rTTrMO7J//3MZJn3QSLF1a9XGDBsHChVbCe999Ex+ncy5leILIJHvvbTMvR42yZqOjjrIFjKINhx01Cp5/Hu67D049NamhOueC5wki04hY9dXZs+Hoo21G9MUXW+mMcAsXWuXY3r2tpIdzLuN4gshU2dnwwQdWrmP8eBsOO2SI1aeqV88eA7z4oi2B6pzLOJ4gMllWlo1K+vRTu7J4+GGb56AK27dbXacPPww6SudcQDxBOGtqysqquH3r1rgW/nLO1S2eIJwpKoq8PY6Fv5xzdYsnCGfSobKtcy6uPEE4ky6VbZ1zcZPQBCEifURkoYgsFpEhEZ4fKCLFIjIzdLs+7LmrReTL0O3qRMbpSL/Kts65WkvYkqMikgUsAs4AioDpwGWqOj9sn4FArqoOKnfs3kAhkAsoMAPorqpror1ejZYcdc65DBfUkqM9gMWqukRVtwFjgL4xHnsW8J6qrg4lhfeAPgmK0znnXASJTBBtgfA18YpC28rrJyKzRWSciLSvzrEicqOIFIpIYXE8F8ZxzjmX0AQRabmx8u1ZbwA5qtoVmAjkV+NYVHW4quaqam7r1q1rFaxzzrndJTJBFAHtwx63A74J30FVV6nq1tDDEUD3WI91zjmXWIlMENOBTiLSUUQaAv2B8eE7iEibsIcXAAtC9ycAZ4pISxFpCZwZ2uaccy5JElaFTVV3iMgg7IM9CxipqvNE5EGgUFXHA4NF5AJgB7AaGBg6drWI/B5LMgAPqurqyl5vxowZP4rI8gT9OsmyD/Bj0EGkEH8/dufvRxl/L3ZXm/cjO9oTCRvm6qpPRAqjDTfLRP5+7M7fjzL+XuwuUe+Hz6R2zjkXkScI55xzEXmCSC3Dgw4gxfj7sTt/P8r4e7G7hLwf3gfhnHMuIr+CcM45F5EnCOeccxF5gkgBItJeRCaJyAIRmScitwUdU9BEJEtEPheRN4OOJWgi0iJUq+yL0L+R44OOKUgickfo/8lcEXlJRBoHHVMyichIEflBROaGbdtbRN4LLY/wXmiCca15gkgNO4C7VPVw4DjgVhE5IuCYgnYbZTPrM90w4B1VPQw4igx+X0SkLTAYWybgSGwSbv9go0q656lY3XoI8L6qdgLeDz2uNU8QKUBVv1XVz0L3N2AfAJEq32YEEWkHnAs8G3QsQRORPYHewD8BVHWbqq4NNqrA1QeaiEh9oCkZVqdNVT/CKk+E60tZsdN84GfxeC1PEClGRHKAo4FPg40kUH8DfgOUBB1ICjgQKAaeCzW5PSsizYIOKiiquhJ41JUtzgAAAzVJREFUFPga+BZYp6rvBhtVSthPVb8F+8IJ7BuPk3qCSCEi0hz4F3C7qq4POp4giMh5wA+qOiPoWFJEfeAY4GlVPRr4iTg1H9RFobb1vkBH4ACgmYhcEWxU6csTRIoQkQZYcihQ1X8HHU+ATgAuEJFl2CqEp4rI6GBDClQRUKSqpVeU47CEkalOB5aqarGqbgf+DfQKOKZU8H1pdezQzx/icVJPEClARARrY16gqn8JOp4gqerdqtpOVXOwzscPVDVjvyGq6nfAChE5NLTpNGB+JYeku6+B40Skaej/zWlkcKd9mPHA1aH7VwOvx+OkCSv37arlBOBKYI6IzAxtu0dV3wowJpc6fgkUhNZVWQJcE3A8gVHVT0VkHPAZNvrvczKs7IaIvAScDOwjIkXA/cCfgJdF5DosiV4cl9fyUhvOOeci8SYm55xzEXmCcM45F5EnCOeccxF5gnDOOReRJwjnnHMReYJwrgoislNEZobd4jaTWURywqtyOpdKfB6Ec1XbrKrdgg7CuWTzKwjnakhElonIwyLyv9Dt4ND2bBF5X0Rmh352CG3fT0ReFZFZoVtpiYgsERkRWuPgXRFpEtp/sIjMD51nTEC/pstgniCcq1qTck1Ml4Y9t15VewBPYFVoCd1/QVW7AgXA46HtjwP/VdWjsHpK80LbOwFPqmpnYC3QL7R9CHB06Dw3J+qXcy4an0ntXBVEZKOqNo+wfRlwqqouCRVb/E5VW4nIj0AbVd0e2v6tqu4jIsVAO1XdGnaOHOC90EIviMhvgQaq+pCIvANsBF4DXlPVjQn+VZ3bjV9BOFc7GuV+tH0i2Rp2fydlfYPnAk8C3YEZoQVynEsaTxDO1c6lYT+nhu5/QtkymAOAKaH77wO3wK41t/eMdlIRqQe0V9VJ2OJJLYAKVzHOJZJ/I3Guak3CquyCrQ9dOtS1kYh8in3Zuiy0bTAwUkR+ja0GV1p99TZgeKji5k4sWXwb5TWzgNEishcgwF99qVGXbN4H4VwNhfogclX1x6BjcS4RvInJOedcRH4F4ZxzLiK/gnDOOReRJwjnnHMReYJwzjkXkScI55xzEXmCcM45F9H/BwRNPU7U/spVAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "\n",
    "def plot_metric(dfhistory, metric):\n",
    "    train_metrics = dfhistory[metric]\n",
    "    val_metrics = dfhistory['val_'+metric]\n",
    "    epochs = range(1, len(train_metrics) + 1)\n",
    "    plt.plot(epochs, train_metrics, 'bo--')\n",
    "    plt.plot(epochs, val_metrics, 'ro-')\n",
    "    plt.title('Training and validation '+ metric)\n",
    "    plt.xlabel(\"Epochs\")\n",
    "    plt.ylabel(metric)\n",
    "    plt.legend([\"train_\"+metric, 'val_'+metric])\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "# 观察损失和准确率的变化\n",
    "plot_metric(dfhistory,\"loss\")\n",
    "plot_metric(dfhistory,\"auc\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "过了第8次之后，开始过拟合了"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T13:35:22.156603Z",
     "start_time": "2020-12-28T13:35:22.141643Z"
    }
   },
   "outputs": [],
   "source": [
    "# 预测\n",
    "y_pred_probs = model(torch.tensor(test_x).float())\n",
    "y_pred = torch.where(y_pred_probs>0.5, torch.ones_like(y_pred_probs), torch.zeros_like(y_pred_probs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T13:36:30.776156Z",
     "start_time": "2020-12-28T13:36:30.756209Z"
    },
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.]])"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T13:37:24.486839Z",
     "start_time": "2020-12-28T13:37:24.458880Z"
    }
   },
   "outputs": [],
   "source": [
    "# 模型的保存与使用\n",
    "torch.save(model, './model/DeepFM.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T13:37:46.232936Z",
     "start_time": "2020-12-28T13:37:46.210995Z"
    }
   },
   "outputs": [],
   "source": [
    "net_clone = torch.load('./model/DeepFM.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T13:38:41.540368Z",
     "start_time": "2020-12-28T13:38:41.527402Z"
    }
   },
   "outputs": [],
   "source": [
    "y_pred_probs = model(torch.tensor(test_x).float())\n",
    "y_pred = torch.where(y_pred_probs>0.5, torch.ones_like(y_pred_probs), torch.zeros_like(y_pred_probs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-28T13:38:47.445771Z",
     "start_time": "2020-12-28T13:38:47.423829Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.]])"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
